Error: EMFILE: too many open files about nodejs-translate

[bcoe]

@chsnt the problem isn't the Translate client, I believe it's this code:

     getFiles(dir, function (err, files) {

          pbSubr.start(files.length, 0);        

            files.forEach( file => {
              fs.pathExists(file.replace('data\\boards\\', 'data\\boards-ru\\'))
              .then(exists => { 
                if(!exists) { 
                    translatorWithErrorHandle (file); 
                } else {                  
                  pbSubr.increment()
                  pbSubr.update(undefined, {dir: file.split('\\')[3]});
                }
              })
                                
            });

        });

This is going to open all files in parallel, so if there are many files in dir you might hit a maximum amount of concurrent open files at the operating system level.

I would potentially rewrite this like:

     getFiles(dir, async function (err, files) {

          pbSubr.start(files.length, 0);        
          for (const file of files) {
              const exists = await fs.pathExists(file.replace('data\\boards\\', 'data\\boards-ru\\'))
              . if(!exists) { 
                  await translatorWithErrorHandle (file); 
                } else {                  
                  pbSubr.increment()
                  await pbSubr.update(undefined, {dir: file.split('\\')[3]});
                }
              })
          } 
        });

☝️ this is pseudo code that would translate a file one at a time, assuming that pbSubr.update returns a promise.

As an alternative to async/await you might also look at p-queue which allows you to control the amount of concurrency happening in your program.

In general, you don't want to perform a large amount of async work in a loop, because you'll create a lot of open handles at once.

[bcoe]

@chsnt this indicates that there are too many open file handles in your application, is there a chance you are initializing the client in a loop? Could you share a snippet of the code you are running?

[chsnt]

@bcoe

main.js

require('dotenv').config({ debug: process.env.GOOGLE_APPLICATION_CREDENTIALS })
const translate = require('./translate');
...
translate (text, targetLang)
...

translate.js

const {Translate} = require('@google-cloud/translate').v2;

const tr = (text, target) => {
  return new Promise ((resolve, reject)=>{
    translate.translate(text, target)
    .then(([result])=>resolve(result))
    .catch(err=>{
        console.error(err); 
        reject(err)
    })
  })
      
}

module.exports = tr;

[bcoe]

@chsnt what does the following do:

require('dotenv').config({ debug: process.env.GOOGLE_APPLICATION_CREDENTIALS })

I could you try running translate.js in isolation, with GOOGLE_APPLICATION_CREDENTIALS set to the path of your credentials file?

[chsnt]

@bcoe

I could you try running translate.js in isolation, with GOOGLE_APPLICATION_CREDENTIALS set to the path of your credentials file?

i run this

require('dotenv').config({ debug: process.env.GOOGLE_APPLICATION_CREDENTIALS })
const {Translate} = require('@google-cloud/translate').v2;

const tr = (text, target) => {
  return new Promise ((resolve, reject)=>{
    translate.translate(text, target)
    .then(([result])=>resolve(result))
    .catch(err=>{
        console.error(err); 
        reject(err)
    })
  })      
}
  
tr('hello', 'ru')
.then(res => console.log(res))
.catch(err => console.log(err))

and it works correctly.

But when i run it without this string

require('dotenv').config({ debug: process.env.GOOGLE_APPLICATION_CREDENTIALS })

i geting this error

Error: Unable to detect a Project Id in the current environment.

[bcoe]

try:

const {Translate} = require('@google-cloud/translate').v2;
const translate = Translate({projectId: 'your-project-id'})
const tr = (text, target) => {
  return new Promise ((resolve, reject)=>{
    translate.translate(text, target)
    .then(([result])=>resolve(result))
    .catch(err=>{
        console.error(err); 
        reject(err)
    })
  })      
}
  
tr('hello', 'ru')
.then(res => console.log(res))
.catch(err => console.log(err))

Providing GOOGLE_APPLICATION_CREDENTIALS with a path to your credentials, rather than using dotenv.

dotenv simply loads a local .env file, with environment variables, I just want to eliminate this as a potential cause of your issues.

[chsnt]

In my system environment variables
GOOGLE_APPLICATION_CREDENTIALS=d:\Projects\translator\translator-22f1b569e00b.json

• translator-22f1b569e00b.json is my key-file (in .env-file : GOOGLE_APPLICATION_CREDENTIALS=D:\Projects\translator\translator-22f1b569e00b.json )

i try this code:

const {Translate} = require('@google-cloud/translate').v2;
const translate = Translate({projectId: 'your-project-id'})
const tr = (text, target) => {
  return new Promise ((resolve, reject)=>{
    translate.translate(text, target)
    .then(([result])=>resolve(result))
    .catch(err=>{
        console.error(err); 
        reject(err)
    })
  })      
}
  
tr('hello', 'ru')
.then(res => console.log(res))
.catch(err => console.log(err))

and got error:

Error: Unable to detect a Project Id in the current environment.    

 at D:\Projects\translator\node_modules\google-auth-library\build\src\auth\googleauth.js:90:31
 at processTicksAndRejections (internal/process/task_queues.js:93:5)

[bcoe]

@chsnt could you also try setting the environment variable GCLOUD_PROJECT to the name of the project the GOOGLE_APPLICATION_CREDENTIALS.

I'm not sure why dotenv would not be throwing the same error, but I believe you might find setting this environment variable explicitly will do the trick.

[chsnt]

@bcoe
Sory, i try it again

const {Translate} = require('@google-cloud/translate').v2;
const translate = Translate({projectId: 'your-project-id'})
const tr = (text, target) => {
  return new Promise ((resolve, reject)=>{
    translate.translate(text, target)
    .then(([result])=>resolve(result))
    .catch(err=>{
        console.error(err); 
        reject(err)
    })
  })      
}
  
tr('hello', 'ru')
.then(res => console.log(res))
.catch(err => console.log(err))

and it works correctly (for single use).

But when i use this code like a module for multiple call i had error again:

EMFILE: too many open files, open 'd:\Projects\translator\translator-22f1b569e00b.json'
[Error: EMFILE: too many open files, open 'd:\Projects\translator\translator-22f1b569e00b.json'] {
  errno: -4066,
  code: 'EMFILE',
  syscall: 'open',
  path: 'd:\\Projects\\translator\\translator-22f1b569e00b.json'
}

(i dont use dotenv anymore)

[bcoe]

@chsnt could you provide code that demonstrates how you are running this logic multiple times; my hunch is that there's a chance you're creating a Translate client in a loop. Make sure you only create a single translate instance, and then call this multiple times (rather than creating a new translate client for each request.

[chsnt]

@bcoe
Thanks for an answer
I use translate like this:

const translateG = require('./translate');
...
translateG (text, targetLang)  // multiple call (its wrong?)
...

(full code https://gist.github.com/chsnt/0d669b0a9509086dce829f8d0ae7257c )

translate.js

const {Translate} = require('@google-cloud/translate').v2;
const translate = Translate({projectId: 'your-project-id'})
const tr = (text, target) => {
  return new Promise ((resolve, reject)=>{
    translate.translate(text, target)
    .then(([result])=>resolve(result))
    .catch(err=>{
        console.error(err); 
        reject(err)
    })
  })      
}

module.exports = tr;