From 5c7b20fc22837d619103c5ded7b21fe4dae900c5 Mon Sep 17 00:00:00 2001 From: Colum Bennett Date: Tue, 14 Mar 2017 14:45:50 +0000 Subject: [PATCH 1/7] add queue module, fileSystem module, fixed eslint errors after running tests --- .eslintrc | 1 + .gitignore | 2 + lib/config.js | 12 +++-- lib/file.js | 82 +++++++++++++++++++++------------ lib/fileSystem.js | 115 ++++++++++++++++++++++++++++++++++++++++++++++ lib/queue.js | 100 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 279 insertions(+), 33 deletions(-) create mode 100644 lib/fileSystem.js create mode 100644 lib/queue.js diff --git a/.eslintrc b/.eslintrc index edbccd78..54024858 100644 --- a/.eslintrc +++ b/.eslintrc @@ -28,6 +28,7 @@ "curly": ["error", "all"] }, "globals": { + "PERSISTENT": false, "angular": false, "$fh": false, "FileTransfer": false, diff --git a/.gitignore b/.gitignore index 05ade97f..176dfecc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ node_modules npm-debug.log .idea +.vscode + diff --git a/lib/config.js b/lib/config.js index 125b41ce..e8f81034 100644 --- a/lib/config.js +++ b/lib/config.js @@ -1,8 +1,14 @@ -'use strict'; +// 'use strict'; module.exports = { apiHost: 'http://localhost:8080', apiPath: '/file/wfm', cloudDataTopicPrefix: 'wfm:cloud:data:', - datasetId: 'file' -}; + datasetId: 'file', + queue: { + prefix: 'fh.wfm.file.queue.', + storageType: 'localStorage', + uploadsLabel: 'uploads', + downloadsLabel: 'downloads' + } +}; \ No newline at end of file diff --git a/lib/file.js b/lib/file.js index 950e0e14..bf6d68b0 100644 --- a/lib/file.js +++ b/lib/file.js @@ -1,6 +1,7 @@ 'use strict'; var config = require('./config'), + QUEUE = require('./queue'), q = require('q'); var client = {}; @@ -25,7 +26,15 @@ client.init = function() { deferredReady.resolve(); } - client.initPromise = q.all([deferredFhinit.promise, deferredReady.promise]); + var deferredInitQueues = q.defer(); + if (!client[config.queue.uploadsLabel]) { + client.uploads = new QUEUE(config.queue.uploadsLabel, config.queue.storageType) || {}; + deferredInitQueues.resolve(); + } else if (client[config.queue.uploadsLabel].ready) { + deferredInitQueues.resolve(); + } + + client.initPromise = q.all([deferredFhinit.promise, deferredReady.promise, deferredInitQueues.promise]); return client.initPromise; }; @@ -35,14 +44,12 @@ client.uploadDataUrl = function(userId, dataUrl) { deferred.reject('Both userId and a dataUrl parameters are required.'); } else { $fh.cloud({ - path: config.apiPath + '/owner/'+userId+'/upload/base64/photo.png', + path: config.apiPath + '/owner/' + userId + '/upload/base64/photo.png', method: 'post', data: dataUrl - }, - function(res) { + }, function(res) { deferred.resolve(res); - }, - function(message, props) { + }, function(message, props) { var e = new Error(message); e.props = props; deferred.reject(e); @@ -52,22 +59,19 @@ client.uploadDataUrl = function(userId, dataUrl) { }; client.list = function(userId) { - var url = arguments.length === 0 ? config.apiPath + '/all' - : config.apiPath + '/owner/' + userId; + var url = arguments.length === 0 ? config.apiPath + '/all' : + config.apiPath + '/owner/' + userId; var deferred = q.defer(); $fh.cloud({ path: url, method: 'get' - }, - function(res) { - deferred.resolve(res); - }, - function(message, props) { - var e = new Error(message); - e.props = props; - deferred.reject(e); - } - ); + }, function(res) { + deferred.resolve(res); + }, function(message, props) { + var e = new Error(message); + e.props = props; + deferred.reject(e); + }); return deferred.promise; }; @@ -84,17 +88,17 @@ function fileUpload(fileURI, serverURI, fileUploadOptions) { function fileUploadRetry(fileURI, serverURI, fileUploadOptions, timeout, retries) { return fileUpload(fileURI, serverURI, fileUploadOptions) - .then(function(response) { - return response; - }, function() { - if (retries === 0) { - throw new Error("Can't upload to " + JSON.stringify(serverURI)); - } - return q.delay(timeout) - .then(function() { - return fileUploadRetry(fileURI, serverURI, fileUploadOptions, timeout, retries - 1); - }); - }); + .then(function(response) { + return response; + }, function() { + if (retries === 0) { + throw new Error("Can't upload to " + JSON.stringify(serverURI)); + } + return q.delay(timeout) + .then(function() { + return fileUploadRetry(fileURI, serverURI, fileUploadOptions, timeout, retries - 1); + }); + }); } client.uploadFile = function(userId, fileURI, options) { @@ -119,6 +123,24 @@ client.uploadFile = function(userId, fileURI, options) { } }; +client.processQueue = function() { + return client.initPromise.then(function() { + return client[config.queue.uploadsLabel].recoverQueue().then(function(processingQueue) { + if (processingQueue) { + client[config.queue.uploadsLabel].queue.forEach(function(item) { + if (window && window.cordova) { + return client.uploadFile(item.userId, item.filePath, { + fileName: item.fileName + }); + } else { + return client.uploadDataUrl(item.userId, item.filePath); + } + }); + } + }); + }); +}; + client.init(); -module.exports = client; +module.exports = client; \ No newline at end of file diff --git a/lib/fileSystem.js b/lib/fileSystem.js new file mode 100644 index 00000000..2dd68a35 --- /dev/null +++ b/lib/fileSystem.js @@ -0,0 +1,115 @@ +var q = require('q'); + +/** + * @param {object} e error object + */ +function _fsErrorHandler(e) { + var msg = ''; + + switch (e.code) { + case FileError.QUOTA_EXCEEDED_ERR: + msg = 'QUOTA_EXCEEDED_ERR'; + break; + case FileError.NOT_FOUND_ERR: + msg = 'NOT_FOUND_ERR'; + break; + case FileError.SECURITY_ERR: + msg = 'SECURITY_ERR'; + break; + case FileError.INVALID_MODIFICATION_ERR: + msg = 'INVALID_MODIFICATION_ERR'; + break; + case FileError.INVALID_STATE_ERR: + msg = 'INVALID_STATE_ERR'; + break; + default: + msg = 'Unknown Error'; + break; + } + + throw new Error('Error: ' + msg); +} + +function _requestFileSystem() { + var d = q.defer(); + navigator.webkitPersistentStorage.requestQuota(1024 * 1024, function(grantedBytes) { + window.webkitRequestFileSystem(PERSISTENT, grantedBytes, function(fs) { + d.resolve(fs); + }, _fsErrorHandler); + }, function(e) { + d.reject(e); + }); + return d.promise; +} + +var fileSystem = {}; + +fileSystem.init = function() { + return _requestFileSystem().then(function(fs) { + fileSystem.ready = true; + fileSystem.fs = fs; + }, function(err) { + fileSystem.ready = false; + throw new Error('Cannot reserve space on fileSystem' + err); + }); +}; + +/** + * @param {string} name fileName/key + * @param {object} data queue data + */ +fileSystem.writeFile = function(name, contentType, data) { + var deferredWrite = q.defer(); + var fileData = JSON.stringify(data); + + if (fileSystem.ready) { + fileSystem.fs.root.getFile(name, { + create: true + }, function(fileEntry) { + // Create a FileWriter object for our FileEntry (log.txt). + fileEntry.createWriter(function(fileWriter) { + fileWriter.onwriteend = function() { + deferredWrite.resolve(true); + }; + fileWriter.onerror = function(err) { + deferredWrite.reject('Write failed: ' + err.toString()); + }; + // Create a new Blob and write it to log.txt. + var blob = new Blob([fileData], { type: contentType }); + fileWriter.write(blob); + }, _fsErrorHandler); + }, _fsErrorHandler); + } + + return deferredWrite.promise; +}; + +/** + * @param {string} name fileName/key + */ +fileSystem.readFile = function(name) { + var deferredRead = q.defer(); + + if (fileSystem.ready) { + fileSystem.fs.root.getFile(name, { + create: false + }, function(fileEntry) { + fileEntry.file(function(file) { + var reader = new FileReader(); + reader.onloadend = function() { + deferredRead.resolve(this.result); + }; + reader.onerror = function(err) { + deferredRead.reject('Unable to read '+ name +' file, Error: ' + err.toString()); + }; + reader.readAsText(file); + }); + }); + } + + return deferredRead.promise; +}; + +fileSystem.init(); + +module.exports = fileSystem; \ No newline at end of file diff --git a/lib/queue.js b/lib/queue.js new file mode 100644 index 00000000..217b6a5b --- /dev/null +++ b/lib/queue.js @@ -0,0 +1,100 @@ +'use strict'; + +var config = require('./config'), + q = require('q'), + _ = require('lodash'); + +// meta data item model schema +// { +// id: , +// fileName: , +// filePath: , +// createdTs: , +// uploadedTs: , +// result: , +// userId: +// step: , +// retries: 3 +// status: +// } + +/** + * @param {object} queue array of q items + * @param {object} item meta-data model + */ +function _existsInQueue(queue, item) { + return _.includes(queue, item); +} + +/** + * @param {string} type + * @param {string} cacheType + */ +var QUEUE = function(type, cacheType) { + this.queueName = config.queue.prefix + this.type; + this.storageType = cacheType !== undefined ? cacheType : config.storageType; + this.queue = []; + this.ready = true; +}; + +QUEUE.prototype.save = function() { + localStorage.setItem(this.queueName, JSON.stringify({ queue: this.queue })); +}; + +QUEUE.prototype.recoverQueue = function() { + var d = q.defer(); + if (localStorage.getItem(this.queueName) !== null) { + var data = JSON.parse(localStorage.getItem(this.queueName)); + this.queue = data.queue; + d.resolve(true); + } else { + d.reject(false); + } + return d.promise; +}; + +QUEUE.prototype.getStorageType = function() { + return this.cacheType; +}; + +QUEUE.prototype.getCount = function() { + return this.queue.length; +}; + +QUEUE.prototype.getQueueName = function() { + return this.queueName; +}; + +/** + * @param {object} item meta data model + */ +QUEUE.prototype.addItem = function(item) { + if (_existsInQueue(this.queue, item)) { + return; + } + this.queue.push(item); +}; + +/** + * @param {object} item meta data model + */ +QUEUE.prototype.removeItem = function(item) { + if (!_existsInQueue(this.queue, item)) { + return; + } + _.remove(this.queue, item); +}; + +/** + * @param {object} item meta data model + */ +QUEUE.prototype.updateItem = function(id, key, val) { + this.queue.filter(function(item) { + if (item.id === id) { + item[key] = val; + return item; + } + }); +}; + +module.exports = QUEUE; \ No newline at end of file From 5ade86e5e7208257a88f0af77acf22b96c22550f Mon Sep 17 00:00:00 2001 From: Colum Bennett Date: Tue, 14 Mar 2017 17:03:16 +0000 Subject: [PATCH 2/7] convert CRUD ops to promises --- lib/file.js | 28 +++++++++++++------------ lib/queue.js | 59 ++++++++++++++++++++++++++++++++++------------------ 2 files changed, 54 insertions(+), 33 deletions(-) diff --git a/lib/file.js b/lib/file.js index bf6d68b0..506c63b9 100644 --- a/lib/file.js +++ b/lib/file.js @@ -125,19 +125,21 @@ client.uploadFile = function(userId, fileURI, options) { client.processQueue = function() { return client.initPromise.then(function() { - return client[config.queue.uploadsLabel].recoverQueue().then(function(processingQueue) { - if (processingQueue) { - client[config.queue.uploadsLabel].queue.forEach(function(item) { - if (window && window.cordova) { - return client.uploadFile(item.userId, item.filePath, { - fileName: item.fileName - }); - } else { - return client.uploadDataUrl(item.userId, item.filePath); - } - }); - } - }); + return client.uploads.recoverQueue(); + }).then(function(processingQueue) { + if (processingQueue) { + client[config.queue.uploadsLabel].queue.forEach(function(item) { + if (window && window.cordova) { + return client.uploadFile(item.userId, item.filePath, { + fileName: item.fileName + }); + } else { + return client.uploadDataUrl(item.userId, item.filePath); + } + }); + } else { + return; + } }); }; diff --git a/lib/queue.js b/lib/queue.js index 217b6a5b..8ba38f9b 100644 --- a/lib/queue.js +++ b/lib/queue.js @@ -31,20 +31,24 @@ function _existsInQueue(queue, item) { * @param {string} cacheType */ var QUEUE = function(type, cacheType) { - this.queueName = config.queue.prefix + this.type; + this.queueName = config.queue.prefix + type; this.storageType = cacheType !== undefined ? cacheType : config.storageType; this.queue = []; this.ready = true; }; QUEUE.prototype.save = function() { + var d = q.defer(); localStorage.setItem(this.queueName, JSON.stringify({ queue: this.queue })); + d.resolve(); + return d.promise; }; QUEUE.prototype.recoverQueue = function() { var d = q.defer(); - if (localStorage.getItem(this.queueName) !== null) { - var data = JSON.parse(localStorage.getItem(this.queueName)); + var queueData = localStorage.getItem(this.queueName); + if (queueData !== null) { + var data = JSON.parse(queueData); this.queue = data.queue; d.resolve(true); } else { @@ -53,48 +57,63 @@ QUEUE.prototype.recoverQueue = function() { return d.promise; }; -QUEUE.prototype.getStorageType = function() { - return this.cacheType; -}; - -QUEUE.prototype.getCount = function() { - return this.queue.length; -}; - -QUEUE.prototype.getQueueName = function() { - return this.queueName; -}; - /** * @param {object} item meta data model */ QUEUE.prototype.addItem = function(item) { + var d = q.defer(); if (_existsInQueue(this.queue, item)) { - return; + d.reject('Item already added to queue'); + } else { + this.queue.push(item); + d.resolve(true); } - this.queue.push(item); + return d.promise; }; /** * @param {object} item meta data model */ QUEUE.prototype.removeItem = function(item) { + var d = q.defer(); if (!_existsInQueue(this.queue, item)) { - return; + d.reject('Cannot find queue item.'); + } else { + _.remove(this.queue, item); + d.resolve(true); } - _.remove(this.queue, item); + return d.promise; }; /** * @param {object} item meta data model */ QUEUE.prototype.updateItem = function(id, key, val) { + var d = q.defer(); this.queue.filter(function(item) { if (item.id === id) { item[key] = val; - return item; + d.resolve(true); + } else { + d.reject('Queue item no updated'); + } + }); + return d.promise; +}; + +/** + * @param {object} item meta data model + */ +QUEUE.prototype.readItem = function(id) { + var d = q.defer(); + this.queue.filter(function(item) { + if (item.id === id) { + d.resolve(item); + } else { + d.reject('Queue item no updated'); } }); + return d.promise; }; module.exports = QUEUE; \ No newline at end of file From 38a15f1d4780b74adeb73cc132609ca54ea09bf6 Mon Sep 17 00:00:00 2001 From: Colum Bennett Date: Tue, 14 Mar 2017 18:34:22 +0000 Subject: [PATCH 3/7] add fs module to fileClient object --- lib/file.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/file.js b/lib/file.js index 506c63b9..f8f6609a 100644 --- a/lib/file.js +++ b/lib/file.js @@ -2,6 +2,7 @@ var config = require('./config'), QUEUE = require('./queue'), + fsModule = require('./fileSystem.js'), q = require('q'); var client = {}; @@ -34,6 +35,8 @@ client.init = function() { deferredInitQueues.resolve(); } + client.fs = fsModule.fs; + client.initPromise = q.all([deferredFhinit.promise, deferredReady.promise, deferredInitQueues.promise]); return client.initPromise; }; From e3a6fd50fb8d29d9419a44c647270d89f8bef721 Mon Sep 17 00:00:00 2001 From: Colum Bennett Date: Wed, 15 Mar 2017 09:58:48 +0000 Subject: [PATCH 4/7] add else conditional to fileSystem ready check, remove fileError from fsErrorHandler --- lib/fileSystem.js | 30 ++++++------------------------ lib/queue.js | 41 +++++++++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/lib/fileSystem.js b/lib/fileSystem.js index 2dd68a35..6bd4e97f 100644 --- a/lib/fileSystem.js +++ b/lib/fileSystem.js @@ -4,30 +4,8 @@ var q = require('q'); * @param {object} e error object */ function _fsErrorHandler(e) { - var msg = ''; - - switch (e.code) { - case FileError.QUOTA_EXCEEDED_ERR: - msg = 'QUOTA_EXCEEDED_ERR'; - break; - case FileError.NOT_FOUND_ERR: - msg = 'NOT_FOUND_ERR'; - break; - case FileError.SECURITY_ERR: - msg = 'SECURITY_ERR'; - break; - case FileError.INVALID_MODIFICATION_ERR: - msg = 'INVALID_MODIFICATION_ERR'; - break; - case FileError.INVALID_STATE_ERR: - msg = 'INVALID_STATE_ERR'; - break; - default: - msg = 'Unknown Error'; - break; - } - - throw new Error('Error: ' + msg); + console.error(e, e.name); + throw new Error('Error: ' + e); } function _requestFileSystem() { @@ -79,6 +57,8 @@ fileSystem.writeFile = function(name, contentType, data) { fileWriter.write(blob); }, _fsErrorHandler); }, _fsErrorHandler); + } else { + deferredWrite.reject('fileSystem not ready'); } return deferredWrite.promise; @@ -105,6 +85,8 @@ fileSystem.readFile = function(name) { reader.readAsText(file); }); }); + } else { + deferredRead.reject('fileSystem not ready'); } return deferredRead.promise; diff --git a/lib/queue.js b/lib/queue.js index 8ba38f9b..63b5733f 100644 --- a/lib/queue.js +++ b/lib/queue.js @@ -2,6 +2,7 @@ var config = require('./config'), q = require('q'), + fileSystem = require('./fileSystem'), _ = require('lodash'); // meta data item model schema @@ -39,21 +40,45 @@ var QUEUE = function(type, cacheType) { QUEUE.prototype.save = function() { var d = q.defer(); - localStorage.setItem(this.queueName, JSON.stringify({ queue: this.queue })); - d.resolve(); + + if (this.storageType === 'file') { + return fileSystem.writeFile(this.fileName, { queue: this.queue }) + .then(function() { + d.resolve(); + }, function() { + d.reject(); + }); + } else { + localStorage.setItem(this.fileName, JSON.stringify({ queue: this.queue })); + d.resolve(); + } + return d.promise; }; QUEUE.prototype.recoverQueue = function() { + var self = this; var d = q.defer(); - var queueData = localStorage.getItem(this.queueName); - if (queueData !== null) { - var data = JSON.parse(queueData); - this.queue = data.queue; - d.resolve(true); + + if (this.storageType === 'file') { + return fileSystem.readFile(this.fileName).then(function(data) { + var fileData = JSON.parse(data); + self.queue = fileData.queue; + d.resolve(true); + }, function() { + d.reject(false); + }); } else { - d.reject(false); + var queueData = localStorage.getItem(this.queueName); + if (queueData !== null) { + var fileData = JSON.parse(queueData); + this.queue = fileData.queue; + d.resolve(true); + } else { + d.reject(false); + } } + return d.promise; }; From 8f0ea5b1c568c9cb71674f83239377bf9801093c Mon Sep 17 00:00:00 2001 From: Colum Bennett Date: Wed, 15 Mar 2017 13:41:21 +0000 Subject: [PATCH 5/7] PR feedback, fileSystem enhancements, persist queue via file system --- lib/file.js | 18 +++++--- lib/fileSystem.js | 105 +++++++++++++++++++++++++++++++++------------- lib/queue.js | 17 ++++---- 3 files changed, 98 insertions(+), 42 deletions(-) diff --git a/lib/file.js b/lib/file.js index f8f6609a..233ef26c 100644 --- a/lib/file.js +++ b/lib/file.js @@ -27,17 +27,23 @@ client.init = function() { deferredReady.resolve(); } - var deferredInitQueues = q.defer(); - if (!client[config.queue.uploadsLabel]) { + var deferredInitUploadsQueue = q.defer(); + if (!client.uploads) { client.uploads = new QUEUE(config.queue.uploadsLabel, config.queue.storageType) || {}; - deferredInitQueues.resolve(); - } else if (client[config.queue.uploadsLabel].ready) { - deferredInitQueues.resolve(); + client.downloads = new QUEUE(config.queue.downloadsLabel, config.queue.storageType) || {}; + deferredInitUploadsQueue.resolve(); + } else if (client.uploads.ready) { + deferredInitUploadsQueue.resolve(); } client.fs = fsModule.fs; - client.initPromise = q.all([deferredFhinit.promise, deferredReady.promise, deferredInitQueues.promise]); + client.initPromise = q.all([ + deferredFhinit.promise, + deferredReady.promise, + deferredInitUploadsQueue.promise + ]); + return client.initPromise; }; diff --git a/lib/fileSystem.js b/lib/fileSystem.js index 6bd4e97f..1467439e 100644 --- a/lib/fileSystem.js +++ b/lib/fileSystem.js @@ -8,14 +8,42 @@ function _fsErrorHandler(e) { throw new Error('Error: ' + e); } -function _requestFileSystem() { +var _requestedFileSystem = {}; +function _checkEnv() { var d = q.defer(); - navigator.webkitPersistentStorage.requestQuota(1024 * 1024, function(grantedBytes) { - window.webkitRequestFileSystem(PERSISTENT, grantedBytes, function(fs) { - d.resolve(fs); - }, _fsErrorHandler); - }, function(e) { - d.reject(e); + + if (window.requestFileSystem) { + _requestedFileSystem = window.requestFileSystem; + d.resolve(true); + } else if (window.webkitRequestFileSystem) { + _requestedFileSystem = window.webkitRequestFileSystem; + d.resolve(true); + } else { + d.reject(false); + } + + if (window.LocalFileSystem) { + PERSISTENT = window.LocalFileSystem.PERSISTENT; + } else if (window.PERSISTENT) { + PERSISTENT = window.PERSISTENT; + } + + return d.promise; +} + +function _requestFS() { + var d = q.defer(); + _checkEnv().then(function(fileSystemAvailable) { + if (fileSystemAvailable) { + _requestedFileSystem(PERSISTENT, 1024 * 1024, function(fs) { + d.resolve(fs); + }, function(e) { + console.error(e, e.name); + d.reject(e.name); + }); + } else { + d.reject('Unable to retrieve fileSystem.'); + } }); return d.promise; } @@ -23,53 +51,70 @@ function _requestFileSystem() { var fileSystem = {}; fileSystem.init = function() { - return _requestFileSystem().then(function(fs) { - fileSystem.ready = true; + var d = q.defer(); + _requestFS().then(function(fs) { fileSystem.fs = fs; - }, function(err) { - fileSystem.ready = false; - throw new Error('Cannot reserve space on fileSystem' + err); + d.resolve(true); + }, function(errorMsg) { + d.reject(errorMsg); }); + return d.promise; +}; + +/** + * @param {string} name directory name + */ +fileSystem.getDir = function(name) { + var d = q.defer(); + if (fileSystem.ready) { + fileSystem.fs.root.getDirectory(name, { + create: true + }, function(dirEntry) { + d.resolve(dirEntry); + }, function(error) { + d.reject(error.toString()); + }); + } else { + d.reject('fileSystem not ready'); + } + return d.promise; }; /** * @param {string} name fileName/key + * @param {string} contentType file content type * @param {object} data queue data */ fileSystem.writeFile = function(name, contentType, data) { - var deferredWrite = q.defer(); + var d = q.defer(); var fileData = JSON.stringify(data); - if (fileSystem.ready) { fileSystem.fs.root.getFile(name, { create: true }, function(fileEntry) { - // Create a FileWriter object for our FileEntry (log.txt). fileEntry.createWriter(function(fileWriter) { fileWriter.onwriteend = function() { - deferredWrite.resolve(true); + d.resolve(true); }; fileWriter.onerror = function(err) { - deferredWrite.reject('Write failed: ' + err.toString()); + d.reject('Write failed: ' + err.toString()); }; - // Create a new Blob and write it to log.txt. var blob = new Blob([fileData], { type: contentType }); fileWriter.write(blob); }, _fsErrorHandler); }, _fsErrorHandler); } else { - deferredWrite.reject('fileSystem not ready'); + d.reject('fileSystem not ready'); } - return deferredWrite.promise; + return d.promise; }; /** * @param {string} name fileName/key */ fileSystem.readFile = function(name) { - var deferredRead = q.defer(); - + var d = q.defer(); if (fileSystem.ready) { fileSystem.fs.root.getFile(name, { create: false @@ -77,21 +122,25 @@ fileSystem.readFile = function(name) { fileEntry.file(function(file) { var reader = new FileReader(); reader.onloadend = function() { - deferredRead.resolve(this.result); + d.resolve(this.result); }; reader.onerror = function(err) { - deferredRead.reject('Unable to read '+ name +' file, Error: ' + err.toString()); + d.reject('Unable to read '+ name +' file, Error: ' + err.toString()); }; reader.readAsText(file); }); }); } else { - deferredRead.reject('fileSystem not ready'); + d.reject('fileSystem not ready!'); } - - return deferredRead.promise; + return d.promise; }; -fileSystem.init(); +// init fieSysten module +fileSystem.init().then(function(ready) { + fileSystem.ready = ready; +}, function(errorMsg) { + console.error(errorMsg); +}); module.exports = fileSystem; \ No newline at end of file diff --git a/lib/queue.js b/lib/queue.js index 63b5733f..f7ec2a97 100644 --- a/lib/queue.js +++ b/lib/queue.js @@ -42,12 +42,13 @@ QUEUE.prototype.save = function() { var d = q.defer(); if (this.storageType === 'file') { - return fileSystem.writeFile(this.fileName, { queue: this.queue }) - .then(function() { - d.resolve(); - }, function() { - d.reject(); - }); + return fileSystem.writeFile(this.fileName, 'application/json', { + queue: this.queue + }).then(function() { + d.resolve(); + }, function() { + d.reject(); + }); } else { localStorage.setItem(this.fileName, JSON.stringify({ queue: this.queue })); d.resolve(); @@ -120,7 +121,7 @@ QUEUE.prototype.updateItem = function(id, key, val) { item[key] = val; d.resolve(true); } else { - d.reject('Queue item no updated'); + d.reject('Queue item not updated'); } }); return d.promise; @@ -135,7 +136,7 @@ QUEUE.prototype.readItem = function(id) { if (item.id === id) { d.resolve(item); } else { - d.reject('Queue item no updated'); + d.reject('Unable to read queue item'); } }); return d.promise; From 6a74ddb285b1ee6f6cb6a20432e21bdec3eef71e Mon Sep 17 00:00:00 2001 From: Colum Bennett Date: Thu, 16 Mar 2017 09:20:43 +0000 Subject: [PATCH 6/7] implement download queue, updated readme, added basic unit tests --- README.md | 33 ++++++++++++++++++-- lib/file.js | 21 ++++++++----- lib/queue-spec.js | 78 +++++++++++++++++++++++++++++++++++++++++++++++ lib/queue.js | 2 +- 4 files changed, 124 insertions(+), 10 deletions(-) create mode 100644 lib/queue-spec.js diff --git a/README.md b/README.md index 81af4126..0213f06e 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,35 @@ camera.capture() }); ``` +Example of re-init/restarting queues : + +```javascript +fileClient.processQueue().then(function(hasJobs) { + console.info('Found job to process in queues, starting file-Transfer'); +}, function() { + console.info('No jobs to process currently, file-Transfer idle.'); +}) +``` + +Example of adding item to `uploads` queue : + +```javascript +camera.capture() + .then(function(dataUrl) { + return fileClient.uploads.addItem({ + id: profileData.id, + fileName: 'my-super-img-file.png' + fileURi: dataUrl, + createdTs: Date.now(), + status: 'waiting', + retries: 3, + userId: 'trever', + step: 'rsik-assessment', + result: this.model + }) + }); +``` + For a more complete example around files operations, please check the [demo mobile app](https://github.com/feedhenry-raincatcher/raincatcher-demo-mobile/blob/master/src/app/file/file.js). #### Directives @@ -95,8 +124,8 @@ Base url : `/file/wfm` ## Mediator events the module publishes following topics: -- `wfm:file:detail:close` - on file detail view close. - +- `wfm:file:detail:close` - on file detail view close. + Client app example: ``` mediator.subscribeForScope('wfm:file:detail:close', $scope, function() { diff --git a/lib/file.js b/lib/file.js index 233ef26c..5d966ad2 100644 --- a/lib/file.js +++ b/lib/file.js @@ -30,18 +30,26 @@ client.init = function() { var deferredInitUploadsQueue = q.defer(); if (!client.uploads) { client.uploads = new QUEUE(config.queue.uploadsLabel, config.queue.storageType) || {}; - client.downloads = new QUEUE(config.queue.downloadsLabel, config.queue.storageType) || {}; deferredInitUploadsQueue.resolve(); } else if (client.uploads.ready) { deferredInitUploadsQueue.resolve(); } + var deferredInitDownloadsQueue = q.defer(); + if (!client.downloads) { + client.downloads = new QUEUE(config.queue.downloadsLabel, config.queue.storageType) || {}; + deferredInitDownloadsQueue.resolve(); + } else if (client.downloads.ready) { + deferredInitUploadsQueue.resolve(); + } + client.fs = fsModule.fs; client.initPromise = q.all([ deferredFhinit.promise, deferredReady.promise, - deferredInitUploadsQueue.promise + deferredInitUploadsQueue.promise, + deferredInitDownloadsQueue.promise ]); return client.initPromise; @@ -103,10 +111,9 @@ function fileUploadRetry(fileURI, serverURI, fileUploadOptions, timeout, retries if (retries === 0) { throw new Error("Can't upload to " + JSON.stringify(serverURI)); } - return q.delay(timeout) - .then(function() { - return fileUploadRetry(fileURI, serverURI, fileUploadOptions, timeout, retries - 1); - }); + return q.delay(timeout).then(function() { + return fileUploadRetry(fileURI, serverURI, fileUploadOptions, timeout, retries - 1); + }); }); } @@ -137,7 +144,7 @@ client.processQueue = function() { return client.uploads.recoverQueue(); }).then(function(processingQueue) { if (processingQueue) { - client[config.queue.uploadsLabel].queue.forEach(function(item) { + client.uploads.queue.forEach(function(item) { if (window && window.cordova) { return client.uploadFile(item.userId, item.filePath, { fileName: item.fileName diff --git a/lib/queue-spec.js b/lib/queue-spec.js new file mode 100644 index 00000000..3ce138dd --- /dev/null +++ b/lib/queue-spec.js @@ -0,0 +1,78 @@ +var assert = require('assert'), + _ = require('lodash'); + +describe('Queue', function() { + var queue = {}, + fileTestQ = {}, + lsTestQ = {}; + + var item = { + 'id': 1, + 'hello': 'world', + 'date': Date.now() + }; + + before(function() { + queue = require('./queue'); + lsTestQ = new queue('lsTestQ', 'localStorage'); + fileTestQ = new queue('fileTestQ', 'file'); + }); + + it('should return new queue with name lsTestQ', function() { + assert.equal(lsTestQ.queueName, 'fh.wfm.file.queue.lsTestQ'); + }); + + it('should return new queue with name fileTestQ', function() { + assert.equal(fileTestQ.queueName, 'fh.wfm.file.queue.fileTestQ'); + }); + + it('lsTestQ should have storageType of localStorage', function() { + assert.equal(lsTestQ.storageType, 'localStorage'); + }); + + it('fileTestQ should have storageType of file', function() { + assert.equal(fileTestQ.storageType, 'file'); + }); + + it('should allow item to be added to lsTestQ queue array', function() { + lsTestQ.addItem(item); + var lsExists = _.includes(lsTestQ.queue, item); + assert.equal(lsExists, true); + + fileTestQ.addItem(item); + var fileExists = _.includes(fileTestQ.queue, item); + assert.equal(fileExists, true); + }); + + it('should get response equal to item object from both queues', function() { + lsTestQ.readItem().then(function(res) { + assert.equal(res, item); + }); + + fileTestQ.readItem().then(function(res) { + assert.equal(res, item); + }); + }); + + it('should update hello prop value to `update test`', function() { + lsTestQ.updateItem(1, 'hello', 'update test'); + lsTestQ.readItem().then(function(item) { + assert.equal(item.hello, 'update test'); + }); + + fileTestQ.updateItem(1, 'hello', 'update test'); + fileTestQ.readItem().then(function(item) { + assert.equal(item.hello, 'update test'); + }); + }); + + it('should remove item from both arrays', function() { + lsTestQ.removeItem(item).then(function() { + assert.equal(lsTestQ.queue.length, 0); + }); + + fileTestQ.removeItem(item).then(function() { + assert.equal(fileTestQ.queue.length, 0); + }); + }); +}); \ No newline at end of file diff --git a/lib/queue.js b/lib/queue.js index f7ec2a97..8999ddc1 100644 --- a/lib/queue.js +++ b/lib/queue.js @@ -5,7 +5,7 @@ var config = require('./config'), fileSystem = require('./fileSystem'), _ = require('lodash'); -// meta data item model schema +// example meta data item model schema // { // id: , // fileName: , From 2c60e8d254675fb1e650341142f2c6f18d5201c0 Mon Sep 17 00:00:00 2001 From: Colum Bennett Date: Thu, 16 Mar 2017 09:35:22 +0000 Subject: [PATCH 7/7] set fileContent type from config, add save queues fn to fileClient, update README --- README.md | 36 ++++++++++++++++++++++++++++++------ lib/config.js | 3 ++- lib/file.js | 10 ++++++++++ lib/queue.js | 12 ++++++------ 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 0213f06e..bc3aab01 100644 --- a/README.md +++ b/README.md @@ -42,16 +42,40 @@ camera.capture() }); ``` -Example of re-init/restarting queues : +Example of re-init/restarting fileTransfer queues : ```javascript -fileClient.processQueue().then(function(hasJobs) { - console.info('Found job to process in queues, starting file-Transfer'); -}, function() { - console.info('No jobs to process currently, file-Transfer idle.'); -}) +fileClient.processQueue() + .then(function(hasJobs) { + console.info('Found job to process in queues, starting file-Transfer'); + }, function() { + console.info('No jobs to process currently, file-Transfer idle.'); + }) ``` +Example of saving/persisting fileTransfer queues : + +```javascript +fileClient.persistQueue() + .then(function(success) { + console.info('file-Transfer queues have been saved!'); + }, function(error) { + console.error('Problem saving fileTransfer queues!: ', error); + }); +``` + +Example of saving/persisting fileTransfer queues : + +```javascript +fileClient.persistQueue() + .then(function(success) { + console.info('file-Transfer queues have been saved!'); + }, function(error) { + console.error('Problem saving fileTransfer queues!: ', error); + }); +``` + + Example of adding item to `uploads` queue : ```javascript diff --git a/lib/config.js b/lib/config.js index e8f81034..3a13d692 100644 --- a/lib/config.js +++ b/lib/config.js @@ -9,6 +9,7 @@ module.exports = { prefix: 'fh.wfm.file.queue.', storageType: 'localStorage', uploadsLabel: 'uploads', - downloadsLabel: 'downloads' + downloadsLabel: 'downloads', + fileContentType: 'application/json' } }; \ No newline at end of file diff --git a/lib/file.js b/lib/file.js index 5d966ad2..2258231e 100644 --- a/lib/file.js +++ b/lib/file.js @@ -159,6 +159,16 @@ client.processQueue = function() { }); }; +client.persistQueue = function() { + var d = q.defer(); + client.uploads.save().then(function(success) { + d.resolve(success); + }, function(error) { + d.reject(error); + }); + return d.promise; +}; + client.init(); module.exports = client; \ No newline at end of file diff --git a/lib/queue.js b/lib/queue.js index 8999ddc1..e67f2303 100644 --- a/lib/queue.js +++ b/lib/queue.js @@ -42,16 +42,16 @@ QUEUE.prototype.save = function() { var d = q.defer(); if (this.storageType === 'file') { - return fileSystem.writeFile(this.fileName, 'application/json', { + return fileSystem.writeFile(this.fileName, config.queue.fileContentType, { queue: this.queue - }).then(function() { - d.resolve(); - }, function() { - d.reject(); + }).then(function(success) { + d.resolve(success); + }, function(error) { + d.reject(error); }); } else { localStorage.setItem(this.fileName, JSON.stringify({ queue: this.queue })); - d.resolve(); + d.resolve(true); } return d.promise;