From 13e6ac59a90623619ba489a749b3127e124728be Mon Sep 17 00:00:00 2001 From: Mo Bitar Date: Tue, 24 Jan 2017 20:55:55 -0600 Subject: [PATCH] sync providers wip --- .../app/frontend/controllers/header.js | 34 ++- .../app/frontend/models/app/extension.js | 6 +- .../app/frontend/models/sync/syncProvider.js | 36 ++++ .../javascripts/app/services/apiController.js | 202 +++++++++++++----- app/assets/stylesheets/app/_header.scss | 12 +- .../templates/frontend/header.html.haml | 46 ++-- 6 files changed, 258 insertions(+), 78 deletions(-) create mode 100644 app/assets/javascripts/app/frontend/models/sync/syncProvider.js diff --git a/app/assets/javascripts/app/frontend/controllers/header.js b/app/assets/javascripts/app/frontend/controllers/header.js index 61ed0f955..2d8164a83 100644 --- a/app/assets/javascripts/app/frontend/controllers/header.js +++ b/app/assets/javascripts/app/frontend/controllers/header.js @@ -71,15 +71,47 @@ angular.module('app.frontend') }) } + this.syncProviderActionIsEnabled = function(action) { + var provider = apiController.syncProviderForURL(action.url); + if(!provider) { + return null; + } + return provider.status; + } + + this.enableSyncProvider = function(action, extension, primary) { + if(extension.encrypted && !extension.ek) { + alert("You must set an encryption key for this extension before enabling this action."); + return; + } + var provider = apiController.findOrCreateSyncProviderForUrl(action.url); + provider.primary = primary; + provider.enabled = true; + provider.ek = extension.ek; + apiController.addSyncProvider(provider); + } + + this.disableSyncProvider = function(action, extension) { + apiController.removeSyncProvider(apiController.syncProviderForURL(action.url)); + } + this.deleteExtension = function(extension) { if(confirm("Are you sure you want to delete this extension?")) { extensionManager.deleteExtension(extension); + var syncProviderAction = extension.syncProviderAction; + if(syncProviderAction) { + apiController.removeSyncProvider(apiController.syncProviderForURL(syncProviderAction.url)); + } } } this.reloadExtensionsPressed = function() { - if(confirm("For your security, reloading extensions will disable any currently enabled repeat actions.")) { + if(confirm("For your security, reloading extensions will disable any currently enabled sync providers and repeat actions.")) { extensionManager.refreshExtensionsFromServer(); + var syncProviderAction = extension.syncProviderAction; + if(syncProviderAction) { + apiController.removeSyncProvider(apiController.syncProviderForURL(syncProviderAction.url)); + } } } diff --git a/app/assets/javascripts/app/frontend/models/app/extension.js b/app/assets/javascripts/app/frontend/models/app/extension.js index acd42b4e2..2e44cfd09 100644 --- a/app/assets/javascripts/app/frontend/models/app/extension.js +++ b/app/assets/javascripts/app/frontend/models/app/extension.js @@ -56,9 +56,13 @@ class Extension extends Item { this.content_type = "Extension"; } + syncProviderAction() { + return _.find(this.actions, {sync_provider: true}) + } + actionsInGlobalContext() { return this.actions.filter(function(action){ - return action.context == "global"; + return action.context == "global" || action.sync_provider == true; }) } diff --git a/app/assets/javascripts/app/frontend/models/sync/syncProvider.js b/app/assets/javascripts/app/frontend/models/sync/syncProvider.js new file mode 100644 index 000000000..115ec20b1 --- /dev/null +++ b/app/assets/javascripts/app/frontend/models/sync/syncProvider.js @@ -0,0 +1,36 @@ +class SyncProvider { + constructor(obj) { + _.merge(this, obj); + } + + addPendingItems(items) { + if(!this.pendingItems) { + this.pendingItems = []; + } + + this.pendingItems = this.pendingItems.concat(items); + } + + removePendingItems(items) { + this.pendingItems = _.difference(this.pendingItems, items); + } + + get status() { + if(!this.enabled) { + return null; + } + + if(this.primary) return "primary"; + else return "secondary"; + } + + asJSON() { + return { + enabled: this.enabled, + url: this.url, + encrypted: this.encrypted, + ek: this.ek + } + } + +} diff --git a/app/assets/javascripts/app/services/apiController.js b/app/assets/javascripts/app/services/apiController.js index 7c4e163b1..f7ea39895 100644 --- a/app/assets/javascripts/app/services/apiController.js +++ b/app/assets/javascripts/app/services/apiController.js @@ -201,9 +201,71 @@ angular.module('app.frontend') /* - Items + Sync */ + this.syncProviderForURL = function(url) { + return _.find(this.syncProviders, {url: url}); + } + + this.findOrCreateSyncProviderForUrl = function(url) { + var provider = _.find(this.syncProviders, {url: url}); + if(!provider) { + provider = new SyncProvider({url: url}) + } + return provider; + } + + this.setEncryptionStatusForProviderURL = function(providerURL, encrypted) { + this.providerForURL(providerURL).encrypted = encrypted; + this.persistSyncProviders(); + } + + this.loadSyncProviders = function() { + var providers = []; + var saved = localStorage.getItem("syncProviders"); + if(saved) { + var parsed = JSON.parse(saved); + for(var p of parsed) { + providers.push(new SyncProvider(p)); + } + } else { + // no providers saved, use default + if(this.isUserSignedIn()) { + var defaultProvider = new SyncProvider(this.getServer() + "/items/sync", true); + providers.push(defaultProvider); + } + } + + this.syncProviders = providers; + } + this.loadSyncProviders(); + + this.addSyncProvider = function(syncProvider) { + if(syncProvider.primary) { + for(var provider of this.syncProviders) { + provider.primary = false; + } + } + + // since we're adding a new provider, we need to send it EVERYTHING we have now. + syncProvider.addPendingItems(modelManager.allItems); + + this.syncProviders.push(syncProvider); + this.persistSyncProviders(); + } + + this.removeSyncProvider = function(provider) { + _.pull(this.syncProviders, provider); + this.persistSyncProviders(); + } + + this.persistSyncProviders = function() { + localStorage.setItem("syncProviders", JSON.stringify(_.map(this.syncProviders, function(provider) { + return provider.asJSON() + }))); + } + this.setSyncToken = function(syncToken) { this.syncToken = syncToken; localStorage.setItem("syncToken", this.syncToken); @@ -211,14 +273,6 @@ angular.module('app.frontend') this.syncWithOptions = function(callback, options = {}) { - if(this.syncOpInProgress) { - // will perform anoter sync after current completes - this.repeatSync = true; - return; - } - - this.syncOpInProgress = true; - var allDirtyItems = modelManager.getDirtyItems(); // we want to write all dirty items to disk only if the user is not signed in, or if the sync op fails @@ -243,55 +297,94 @@ angular.module('app.frontend') modelManager.clearDirtyItems(allDirtyItems); }.bind(this)) + this.syncOpInProgress = false; - if(callback) { + + if(this.syncProviders.length == 0 && callback) { callback(); } + } + + for(let provider of this.syncProviders) { + if(provider.enabled == false) { + continue; + } + provider.addPendingItems(allDirtyItems); + this.__performSyncWithProvider(provider, options, function(response){ + if(provider.primary) { + if(callback) { + callback(response) + } + } + }) + } + + modelManager.clearDirtyItems(allDirtyItems); + } + + this.__performSyncWithProvider = function(provider, options, callback) { + if(provider.syncOpInProgress) { + provider.repeatOnCompletion = true; + console.log("Sync op in progress for provider; returning.", provider); return; } + + provider.syncOpInProgress = true; + let submitLimit = 100; - var dirtyItems = allDirtyItems.slice(0, submitLimit); - if(dirtyItems.length < allDirtyItems.length) { + var allItems = provider.pendingItems; + var subItems = allItems.slice(0, submitLimit); + if(subItems.length < allItems.length) { // more items left to be synced, repeat - this.repeatSync = true; + provider.repeatOnCompletion = true; } else { - this.repeatSync = false; + provider.repeatOnCompletion = false; } - var request = Restangular.one("items/sync"); + console.log("Syncing with provider", provider, subItems); + + // Remove dirty items now. If this operation fails, we'll re-add them. + // This allows us to queue changes on the same item + provider.removePendingItems(subItems); + + var request = Restangular.oneUrl(provider.url, provider.url); request.limit = 150; - request.sync_token = this.syncToken; - request.cursor_token = this.cursorToken; - request.items = _.map(dirtyItems, function(item){ - return this.createRequestParamsForItem(item, options.additionalFields); + request.items = _.map(subItems, function(item){ + return this.paramsForItem(item, provider.encrypted, provider.ek, options.additionalFields, false); }.bind(this)); + if(provider.primary) { + // only primary providers receive items (or care about received items) + request.sync_token = this.syncToken; + request.cursor_token = provider.cursorToken; + } + request.post().then(function(response) { - modelManager.clearDirtyItems(dirtyItems); + if(provider.primary) { + // handle sync token + this.setSyncToken(response.sync_token); + $rootScope.$broadcast("sync:updated_token", this.syncToken); - // handle sync token - this.setSyncToken(response.sync_token); - $rootScope.$broadcast("sync:updated_token", this.syncToken); + // handle cursor token (more results waiting, perform another sync) + provider.cursorToken = response.cursor_token; - // handle cursor token (more results waiting, perform another sync) - this.cursorToken = response.cursor_token; + var retrieved = this.handleItemsResponse(response.retrieved_items, null, provider); + // merge only metadata for saved items + var omitFields = ["content", "auth_hash"]; + var saved = this.handleItemsResponse(response.saved_items, omitFields, provider); - var retrieved = this.handleItemsResponse(response.retrieved_items, null); - // merge only metadata for saved items - var omitFields = ["content", "auth_hash"]; - var saved = this.handleItemsResponse(response.saved_items, omitFields); + this.handleUnsavedItemsResponse(response.unsaved, provider) - this.handleUnsavedItemsResponse(response.unsaved) + this.writeItemsToLocalStorage(saved, null); + this.writeItemsToLocalStorage(retrieved, null); + } - this.writeItemsToLocalStorage(saved, null); - this.writeItemsToLocalStorage(retrieved, null); + provider.syncOpInProgress = false; - this.syncOpInProgress = false; - - if(this.cursorToken || this.repeatSync == true) { - this.syncWithOptions(callback, options); + if(provider.cursorToken || provider.repeatOnCompletion == true) { + this.__performSyncWithProvider(provider, options, callback); } else { if(callback) { callback(response); @@ -302,8 +395,13 @@ angular.module('app.frontend') .catch(function(response){ console.log("Sync error: ", response); - writeAllDirtyItemsToDisk(); - this.syncOpInProgress = false; + // Re-add subItems since this operation failed. We'll have to try again. + provider.addPendingItems(subItems); + provider.syncOpInProgress = false; + + if(provider.primary) { + this.writeItemsToLocalStorage(allItems, null); + } if(callback) { callback({error: "Sync error"}); @@ -315,7 +413,7 @@ angular.module('app.frontend') this.syncWithOptions(callback, undefined); } - this.handleUnsavedItemsResponse = function(unsaved) { + this.handleUnsavedItemsResponse = function(unsaved, provider) { if(unsaved.length == 0) { return; } @@ -332,19 +430,15 @@ angular.module('app.frontend') } } - this.syncWithOptions(null, {additionalFields: ["created_at", "updated_at"]}); + this.__performSyncWithProvider(provider, {additionalFields: ["created_at", "updated_at"]}, null); } - this.handleItemsResponse = function(responseItems, omitFields) { - this.decryptItems(responseItems); + this.handleItemsResponse = function(responseItems, omitFields, syncProvider) { + this.decryptItemsWithKey(responseItems, syncProvider ? syncProvider.ek : null); return modelManager.mapResponseItemsToLocalModelsOmittingFields(responseItems, omitFields); } - this.createRequestParamsForItem = function(item, additionalFields) { - return this.paramsForItem(item, true, additionalFields, false); - } - - this.paramsForExportFile = function(item, encrypted) { + this.paramsForExportFile = function(item, ek, encrypted) { return _.omit(this.paramsForItem(item, encrypted, ["created_at", "updated_at"], true), ["deleted"]); } @@ -352,15 +446,18 @@ angular.module('app.frontend') return _.omit(this.paramsForItem(item, encrypted, ["created_at", "updated_at"], true), ["deleted"]); } - this.paramsForItem = function(item, encrypted, additionalFields, forExportFile) { + this.paramsForItem = function(item, encrypted, ek, additionalFields, forExportFile) { var itemCopy = _.cloneDeep(item); + if(encrypted) { + console.assert(ek.length, "Attempting to encrypt without encryption key."); + } console.assert(!item.dummy, "Item is dummy, should not have gotten here.", item.dummy) var params = {uuid: item.uuid, content_type: item.content_type, deleted: item.deleted}; if(encrypted) { - this.encryptSingleItem(itemCopy, this.retrieveMk()); + this.encryptSingleItem(itemCopy, ek); params.content = itemCopy.content; params.enc_item_key = itemCopy.enc_item_key; params.auth_hash = itemCopy.auth_hash; @@ -475,7 +572,7 @@ angular.module('app.frontend') this.loadLocalItems = function(callback) { var params = dbManager.getAllItems(function(items){ - var items = this.handleItemsResponse(items, null); + var items = this.handleItemsResponse(items, null, null); Item.sortItemsByDate(items); callback(items); }.bind(this)) @@ -560,11 +657,6 @@ angular.module('app.frontend') item.content = content; } - this.decryptItems = function(items) { - var masterKey = this.retrieveMk(); - this.decryptItemsWithKey(items, masterKey); - } - this.decryptItemsWithKey = function(items, key) { for (var item of items) { if(item.deleted == true) { diff --git a/app/assets/stylesheets/app/_header.scss b/app/assets/stylesheets/app/_header.scss index 5e5e5208f..c888e18d9 100644 --- a/app/assets/stylesheets/app/_header.scss +++ b/app/assets/stylesheets/app/_header.scss @@ -312,7 +312,7 @@ Extensions font-weight: bold !important; } - > .name { + .extension-name { font-weight: bold; font-size: 16px; margin-bottom: 6px; @@ -331,12 +331,12 @@ Extensions } } - > .subtitle { + .extension-subtitle { font-size: 14px; margin-bottom: 10px; } - > .actions { + .extension-actions { margin-top: 15px; font-size: 12px; @@ -346,18 +346,18 @@ Extensions background-color: rgba(white, 0.9); border: 1px solid rgba(gray, 0.15); - > .name { + .action-name { font-weight: bold; } - > .permissions { + .action-permissions { margin-top: 2px; a { font-weight: normal !important; } } - > .execute { + .execute { font-weight: bold; margin-bottom: 0px; font-size: 12px; diff --git a/app/assets/templates/frontend/header.html.haml b/app/assets/templates/frontend/header.html.haml index 5bdc562cb..ad14a0fee 100644 --- a/app/assets/templates/frontend/header.html.haml +++ b/app/assets/templates/frontend/header.html.haml @@ -103,7 +103,7 @@ %div{"style" => "font-size: 18px;", "ng-if" => "!ctrl.extensionManager.extensions.length"} No extensions installed .registered-extensions{"ng-if" => "ctrl.extensionManager.extensions.length"} .extension{"ng-repeat" => "extension in ctrl.extensionManager.extensions"} - .name {{extension.name}} + .extension-name {{extension.name}} .encryption-format .title Send data: %label @@ -112,21 +112,37 @@ %label %input{"type" => "radio", "ng-model" => "extension.encrypted", "ng-value" => "false", "ng-change" => "ctrl.extensionManager.changeExtensionEncryptionFormat(false, extension)"} Decrypted - .actions + .encryption-key{"ng-if" => "extension.encrypted"} + %input{"ng-model" => "extension.ek", "placeholder" => "Set encryption key"} + .extension-actions .action{"ng-repeat" => "action in extension.actionsInGlobalContext()"} - .name {{action.label}} - .desc{"style" => "font-style: italic;"} {{action.desc}} - .execute-type{"ng-if" => "action.repeat_mode == 'watch'"} - Repeats when a change is made to your items. - .execute-type{"ng-if" => "action.repeat_mode == 'loop'"} - Repeats at most once every {{action.repeat_timeout}} seconds - .permissions - %a{"ng-click" => "action.showPermissions = !action.showPermissions"} {{action.showPermissions ? "Hide permissions" : "Show permissions"}} - %div{"ng-if" => "action.showPermissions"} - {{action.permissionsString}} - .encryption-type - %span {{action.encryptionModeString}} - .execute + %div{"ng-if" => "!action.sync_provider"} + .action-name {{action.label}} + .action-desc{"style" => "font-style: italic;"} {{action.desc}} + .execute-type{"ng-if" => "action.repeat_mode == 'watch'"} + Repeats when a change is made to your items. + .execute-type{"ng-if" => "action.repeat_mode == 'loop'"} + Repeats at most once every {{action.repeat_timeout}} seconds + .action-permissions + %a{"ng-click" => "action.showPermissions = !action.showPermissions"} {{action.showPermissions ? "Hide permissions" : "Show permissions"}} + %div{"ng-if" => "action.showPermissions"} + {{action.permissionsString}} + .encryption-type + %span {{action.encryptionModeString}} + %div{"ng-if" => "action.sync_provider"} + .action-name This is a sync provider action. + .action-desc{"style" => "margin-top: -5px;"} + %p Enabling this sync provider as a primary provider will replace your current sync provider. + %p Enabling this sync provider as a secondary provider will save your data with this provider as a backup, but will not be used to pull changes. + %p You can have only one primary provider, and multiple secondary providers. + + %div{"ng-if" => "!ctrl.syncProviderActionIsEnabled(action)"} + .execute{"ng-click" => "ctrl.enableSyncProvider(action, extension, true)"} Enable as primary sync provider + .execute{"ng-click" => "ctrl.enableSyncProvider(action, extension, false)"} Enable as backup sync provider + %div{"ng-if" => "ctrl.syncProviderActionIsEnabled(action)"} + .execute{"ng-click" => "ctrl.disableSyncProvider(action, extension)"} Remove as sync provider + + .execute{"ng-if" => "!action.sync_provider"} %div{"ng-if" => "action.repeat_mode"} %div{"ng-if" => "ctrl.extensionManager.isRepeatActionEnabled(action)", "ng-click" => "ctrl.extensionManager.disableRepeatAction(action, extension)"} Disable %div{"ng-if" => "!ctrl.extensionManager.isRepeatActionEnabled(action)", "ng-click" => "ctrl.extensionManager.enableRepeatAction(action, extension)"} Enable