0

I am using kue to queue and process some python jobs using node express. The redis server i am using is not the default on localhost but AWS Redis. Problem Statement: Connection to AWS redis is successful, jobs are being created and processed, output files are being generated as expected from the python jobs however the Kue JSON API doesn't return any data when the server used is AWS Redis. API returns data perfectly fine if its the redis-server on localhost.

1.Here is my connection code to redis from node

 const redisclient  = redis.createClient({host: hostname, port: 6379});

 var kue = require('kue'), queue = kue.createQueue({
                  prefix: 'q',
                  redis: {
                    port: 6379,
                    host: hostname,
                    options: {
                      disableSearch: false,
                      client: redisclient
                   }
                  }
                });//redis connection works AWS

2.Mounting the Kue UI for Express

var kueUiExpress = require('kue-ui-express');
kueUiExpress(app, '/kue/', '/kue-api/');
app.use('/kue-api/', kue.app);

3.Job Queue and Process Calls

var suspendusershdl = queue.create('suspendusershdl', {username: currentuseremail, title: "Generate HDL file to Suspend Users", inputfile: inputfilepathname, inputcoll: collname, outputfile: outputfilepathname , pythonscript: 'suspendusershdl.py' });
suspendusershdl.searchKeys( ['username'], 'username' ).save(); //indexing by username

suspendusershdl.on('complete', function(result){
                                              console.log('Job is now complete');
                                              console.log('Job Details:', suspendusershdl);
//Additional logic when job is complete
});

queue.process('suspendusershdl', function(suspendusershdl, done){
//process the job, processing logic here
});

4.Here is what I see at this point. If I go to Kue UI all the job details are displayed as expected.

Kue UI Job details displayed as expected

5.Here is what I see on MONITOR on redis-cli for the AWS Redis instance.

1524582499.883115 [0 172.31.31.246:53372] "incr" "q:ids"
1524582499.887225 [0 172.31.31.246:53372] "hset" "q:job:3" "max_attempts" "1"
1524582499.887273 [0 172.31.31.246:53372] "sadd" "q:job:types" "suspendusershdl"
1524582499.887448 [0 172.31.31.246:53372] "hset" "q:job:3" "type" "suspendusershdl"
1524582499.887462 [0 172.31.31.246:53372] "hset" "q:job:3" "created_at" "1524582499888"
1524582499.887470 [0 172.31.31.246:53372] "hset" "q:job:3" "promote_at" "1524582499888"
1524582499.887679 [0 172.31.31.246:53372] "hset" "q:job:3" "updated_at" "1524582499888"
1524582499.887951 [0 172.31.31.246:53372] "hset" "q:job:3" "priority" "0"
1524582499.888013 [0 172.31.31.246:53372] "zadd" "q:jobs" "0" "01|3"
1524582499.888069 [0 172.31.31.246:53372] "hset" "q:job:3" "data" "{\"username\":\"[email protected]\",\"title\":\"Generate HDL file to Suspend Users\",\"inputfile\":\"inputfilepathname",\"inputcoll\":\"suspendusers1524582499883\",\"outputfile\":\"outputfilepathname\",\"pythonscript\":\"suspendusershdl.py\"}"
1524582499.889688 [0 172.31.31.246:53372] "hset" "q:job:3" "updated_at" "1524582499890"
1524582499.889806 [0 172.31.31.246:53372] "multi"
1524582499.889816 [0 172.31.31.246:53372] "hset" "q:job:3" "state" "inactive"
1524582499.889822 [0 172.31.31.246:53372] "zadd" "q:jobs:inactive" "0" "01|3"
1524582499.889829 [0 172.31.31.246:53372] "zadd" "q:jobs:suspendusershdl:inactive" "0" "01|3"
1524582499.889836 [0 172.31.31.246:53372] "lpush" "q:suspendusershdl:jobs" "1"
1524582499.889844 [0 172.31.31.246:53372] "exec"
1524582499.941285 [0 172.31.31.246:53398] "info"
1524582499.941381 [0 172.31.31.246:53400] "info"
1524582499.941507 [0 172.31.31.246:53402] "info"
1524582499.943331 [0 172.31.31.246:53398] "select" "0"
1524582499.943340 [0 172.31.31.246:53398] "blpop" "q:suspendusershdl:jobs" "0"
1524582499.943541 [0 172.31.31.246:53400] "select" "0"
1524582499.943780 [0 172.31.31.246:53402] "select" "0"
1524582499.944132 [0 172.31.31.246:53402] "publish" "q:events" "{\"id\":3,\"event\":\"enqueue\",\"args\":[\"enqueue\",\"suspendusershdl\"]}"
1524582499.945201 [0 172.31.31.246:53372] "multi"
1524582499.945211 [0 172.31.31.246:53372] "zrange" "q:jobs:suspendusershdl:inactive" "0" "0"
1524582499.945220 [0 172.31.31.246:53372] "zremrangebyrank" "q:jobs:suspendusershdl:inactive" "0" "0"
1524582499.945229 [0 172.31.31.246:53372] "exec"
1524582499.947129 [0 172.31.31.246:53402] "hgetall" "q:job:3"
1524582499.949203 [0 172.31.31.246:53372] "hset" "q:job:3" "started_at" "1524582499949"
1524582499.949298 [0 172.31.31.246:53372] "hset" "q:job:3" "workerId" "kue:ip-172-31-31-246:10624:suspendusershdl:1"
1524582499.949467 [0 172.31.31.246:53372] "hset" "q:job:3" "updated_at" "1524582499950"
1524582499.949550 [0 172.31.31.246:53372] "multi"
1524582499.949563 [0 172.31.31.246:53372] "zrem" "q:jobs:inactive" "01|3"
1524582499.949570 [0 172.31.31.246:53372] "zrem" "q:jobs:suspendusershdl:inactive" "01|3"
1524582499.949575 [0 172.31.31.246:53372] "hset" "q:job:3" "state" "active"
1524582499.949580 [0 172.31.31.246:53372] "zadd" "q:jobs:active" "0" "01|3"
1524582499.949596 [0 172.31.31.246:53372] "zadd" "q:jobs:suspendusershdl:active" "0" "01|3"
1524582499.949603 [0 172.31.31.246:53372] "zadd" "q:jobs:active" "0" "01|3"
1524582499.949608 [0 172.31.31.246:53372] "exec"
1524582499.951338 [0 172.31.31.246:53402] "publish" "q:events" "{\"id\":3,\"event\":\"start\",\"args\":[\"start\",\"suspendusershdl\",null]}"
1524582500.005064 [0 172.31.31.246:53372] "rpush" "q:job:3:log" "File Imported Successfully to MongoDB"
1524582500.005121 [0 172.31.31.246:53372] "hset" "q:job:3" "updated_at" "1524582500005"
1524582500.005188 [0 172.31.31.246:53372] "rpush" "q:job:3:log" "Calling Exec Python"
1524582500.005225 [0 172.31.31.246:53372] "hset" "q:job:3" "updated_at" "1524582500005"
1524582500.138182 [0 172.31.31.246:53390] "multi"
1524582500.138202 [0 172.31.31.246:53390] "zinterstore" "q:searchtmpkey" "3" "q:search:word:HLTMN" "q:search:word:RSP" "q:search:word:KM"
1524582500.138219 [0 172.31.31.246:53390] "zrevrange" "q:searchtmpkey" "0" "-1"
1524582500.138224 [0 172.31.31.246:53390] "zremrangebyrank" "q:searchtmpkey" "0" "-1"
1524582500.138230 [0 172.31.31.246:53390] "exec"
1524582500.620289 [0 172.31.31.246:53372] "rpush" "q:job:3:log" "suspendusershdl.py completed successfully."
1524582500.620317 [0 172.31.31.246:53372] "hset" "q:job:3" "updated_at" "1524582500620"
1524582500.620497 [0 172.31.31.246:53372] "hset" "q:job:3" "duration" "672"
1524582500.620592 [0 172.31.31.246:53372] "hset" "q:job:3" "progress" "100"
1524582500.620684 [0 172.31.31.246:53372] "hset" "q:job:3" "updated_at" "1524582500621"
1524582500.620779 [0 172.31.31.246:53372] "multi"
1524582500.620792 [0 172.31.31.246:53372] "zrem" "q:jobs:active" "01|3"
1524582500.620799 [0 172.31.31.246:53372] "zrem" "q:jobs:suspendusershdl:active" "01|3"
1524582500.620806 [0 172.31.31.246:53372] "hset" "q:job:3" "state" "complete"
1524582500.620811 [0 172.31.31.246:53372] "zadd" "q:jobs:complete" "0" "01|3"
1524582500.620821 [0 172.31.31.246:53372] "zadd" "q:jobs:suspendusershdl:complete" "0" "01|3"
1524582500.620828 [0 172.31.31.246:53372] "exec"
1524582500.622509 [0 172.31.31.246:53372] "hincrby" "q:job:3" "attempts" "1"
1524582500.624102 [0 172.31.31.246:53402] "publish" "q:events" "{\"id\":3,\"event\":\"complete\",\"args\":[\"complete\",null,null]}"
1524582500.624224 [0 172.31.31.246:53372] "incrby" "q:stats:work-time" "672"
1524582500.624382 [0 172.31.31.246:53398] "blpop" "q:suspendusershdl:jobs" "0"
1524582500.887372 [0 172.31.31.246:53400] "set" "promotion:lock" "U3dicEfREeiGI0kMM36BPg==" "PX" "2000" "NX"
1524582500.887552 [0 172.31.31.246:53400] "set" "activeJobsTTL:lock" "U3eJgEfREeiGI0kMM36BPg==" "PX" "2000" "NX"
1524582500.889253 [0 172.31.31.246:53372] "zrangebyscore" "q:jobs:delayed" "0" "1524582500889" "LIMIT" "0" "1000"
1524582500.889452 [0 172.31.31.246:53372] "zrangebyscore" "q:jobs:active" "100000" "1524582500890" "LIMIT" "0" "1000"
1524582500.892561 [0 172.31.31.246:53400] "script" "load" "--\n-- Delete a key if content is equal\n--\n-- KEYS[1]   - key\n-- KEYS[2]   - content\nlocal key     = KEYS[1]\nlocal content = ARGV[1]\n\nlocal value = redis.call('get', key)\n\nif value == content then\n  return redis.call('del', key);\nend\n\nreturn 0\n"

6.So far so good. Now I call the JSON API to query all jobs scheduled by a user.

  request({'rejectUnauthorized': false, 'url': 'https://example.com/kue-api/job/search?q=' + currentuseremail, 'method': 'GET'}, function(error, response, body) {
        if (error) {
            console.log(error);
            res.json({ success : false, message : 'Error', jobs: processesfromkue});
        } else {
            processesfromkue = JSON.parse(body);
            console.log('Number of processes in kue for user: ', processesfromkue.length);
            res.json({ success : true, message : 'Kue Jobs retrieved', jobs: processesfromkue}); //This API Call always results in success and the array of jobs returned is empty although there are jobs displayed in the DB and UI
        }
    });

This API call returns data if I am using the default redis on localhost.

Is there anything I can do to have the JSON API return data from AWS Redis? I can't move to Prod if I have to use redis-server on localhost and this behaviour of the JSON API persists when connecting to AWS Redis.

Thanks for your help.

1 Answer 1

0

Are you sure your Redis instance is in the same VPC as your Kue/Node server?

One way to test connection to aws redis would be to login to the server or a server in the same VPC. Install redis-cli and test connection.

You can install redis-cli with these commands:

yum install libjemalloc1 libjemalloc-dev gcc make -y
cd /tmp
wget http://download.redis.io/redis-stable.tar.gz
tar xvzf redis-stable.tar.gz
cd redis-stable
make
cp src/redis-cli /usr/local/bin/
chmod 755 /usr/local/bin/redis-cli

Re-Login, then:

Testing connection:

redis-cli -c -h REDIS.URL.cache.amazonaws.com -p 6379

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.