0041542: Full Master Data Load can be improved by inserting data in batches

diff --git a/web/org.openbravo.mobile.core/source/data/ob-dal.js b/web/org.openbravo.mobile.core/source/data/ob-dal.js
--- a/web/org.openbravo.mobile.core/source/data/ob-dal.js
+++ b/web/org.openbravo.mobile.core/source/data/ob-dal.js
@@ -1388,66 +1388,127 @@
    *
    *
    */
+
   OB.Dal.insertData = function (model, data, success, error, incremental, offset) {
-    OB.Data.localDB.transaction(function (tx) {
-      var props = model.getProperties ? model.getProperties() : model.prototype.properties,
-          filterVal, values, _idx = offset,
-          updateRecord, handleError, insertStatement, filterProps;
-      handleError = function (txError, e) {
-        executeSqlErrorHandler('error', "OB.Dal.insertData: model insert", model.prototype.modelName, txError, e);
-      };
-      updateRecord = function (tx, model, values, active) {
-        var deleteStatement;
-        deleteStatement = model.getDeleteByIdStatement ? model.getDeleteByIdStatement() : "DELETE FROM " + model.prototype.tableName + " WHERE " + model.prototype.propertyMap.id + "=?";
-        tx.executeSql(deleteStatement, [values[0]], function () {
-          if (_.isUndefined(active) || active) {
-            var insertSatement;
-            insertSatement = model.getInsertStatement ? model.getInsertStatement() : model.prototype.insertStatement;
-            tx.executeSql(insertSatement, values, null, function (tx, e) {
-              handleError(tx, e);
-              return true;
+    if (data && data.length > 0) {
+      OB.Data.localDB.transaction(function (tx) {
+        var props = model.getProperties ? model.getProperties() : model.prototype.properties,
+            filterVal, values, _idx = offset,
+            updateRecord, handleError, insertStatement, filterProps;
+        handleError = function (txError, e) {
+          executeSqlErrorHandler('error', "OB.Dal.insertData: model insert", model.prototype.modelName, txError, e);
+        };
+        updateRecord = function (tx, model, values, active) {
+          var deleteStatement;
+          deleteStatement = model.getDeleteByIdStatement ? model.getDeleteByIdStatement() : "DELETE FROM " + model.prototype.tableName + " WHERE " + model.prototype.propertyMap.id + "=?";
+          tx.executeSql(deleteStatement, [values[0]], function () {
+            if (_.isUndefined(active) || active) {
+              var insertSatement;
+              insertSatement = model.getInsertStatement ? model.getInsertStatement() : model.prototype.insertStatement;
+              tx.executeSql(insertSatement, values, null, function (tx, e) {
+                handleError(tx, e);
+                return true;
+              });
+            }
+          }, handleError);
+        };
+        filterProps = model.getFilterProperties ? model.getFilterProperties() : model.prototype.propertiesFilter;
+        if (incremental) {
+          _.each(data, function (item) {
+            values = [];
+            _.each(props, function (prop) {
+              var value, propName = typeof prop === 'string' ? prop : prop.name;
+              if (!propName || '_idx' === propName || '_filter' === propName) {
+                return;
+              }
+              value = item[propName];
+              values.push(value === undefined ? null : value);
             });
-          }
-        }, handleError);
-      };
-      insertStatement = model.getInsertStatement ? model.getInsertStatement() : model.prototype.insertStatement;
-      filterProps = model.getFilterProperties ? model.getFilterProperties() : model.prototype.propertiesFilter;
-      _.each(data, function (item) {
-        values = [];
-        _.each(props, function (prop) {
-          var value, propName = typeof prop === 'string' ? prop : prop.name;
-          if (!propName || '_idx' === propName || '_filter' === propName) {
-            return;
-          }
-          value = item[propName];
-          values.push(value === undefined ? null : value);
-        });
-        if ((model.hasFilter && model.hasFilter()) || model.prototype.propertiesFilter) {
-          filterVal = '';
-          _.each(filterProps, function (prop) {
-            filterVal = OB.UTIL.unAccent(filterVal) + (item[prop] ? (item[prop] + '###') : '');
+            if ((model.hasFilter && model.hasFilter()) || model.prototype.propertiesFilter) {
+              filterVal = '';
+              _.each(filterProps, function (prop) {
+                filterVal = OB.UTIL.unAccent(filterVal) + (item[prop] ? (item[prop] + '###') : '');
+              });
+              values.push(filterVal);
+            }
+            values.push(_idx);
+            updateRecord(tx, model, values, item.active);
+            _idx++;
           });
-          values.push(filterVal);
+        } else {
+          var i, getBatchInsertStatement, getBatchValues, insertBatchData, insertStatementParts, propertiesPart, valuesPart, currentData, batchInsertStatement, batchValues, modelBatchSize = Math.floor(999 / (props.length + 2));
+
+          insertStatementParts = model.getInsertStatement ? model.getInsertStatement() : model.prototype.insertStatement;
+          insertStatementParts = insertStatementParts.split(/values/i);
+
+          propertiesPart = insertStatementParts[0] + "VALUES\n";
+          valuesPart = insertStatementParts[1];
+
+          getBatchInsertStatement = function (dataSize) {
+            var j, newInsert = propertiesPart;
+            for (j = 0; j < dataSize; j++) {
+              newInsert += valuesPart;
+              if (j < (dataSize - 1)) {
+                newInsert += ',\n';
+              }
+            }
+            return newInsert;
+          };
+
+          getBatchValues = function (dataArray) {
+            var valuesArray = [];
+            _.each(dataArray, function (item) {
+              _.each(props, function (prop) {
+                var value, propName = typeof prop === 'string' ? prop : prop.name;
+                if (!propName || '_idx' === propName || '_filter' === propName) {
+                  return;
+                }
+                value = item[propName];
+                valuesArray.push(value === undefined ? null : value);
+              });
+              if ((model.hasFilter && model.hasFilter()) || model.prototype.propertiesFilter) {
+                filterVal = '';
+                _.each(filterProps, function (prop) {
+                  filterVal = OB.UTIL.unAccent(filterVal) + (item[prop] ? (item[prop] + '###') : '');
+                });
+                valuesArray.push(filterVal);
+              }
+              valuesArray.push(_idx);
+              _idx++;
+            });
+            return valuesArray;
+          };
+
+
+          insertBatchData = function (offset) {
+            if (offset >= data.length) {
+              return;
+            } else {
+              currentData = data.slice(offset, offset + modelBatchSize);
+              batchInsertStatement = getBatchInsertStatement(currentData.length);
+              batchValues = getBatchValues(currentData);
+              tx.executeSql(batchInsertStatement, batchValues, null, function (tx, e) {
+                handleError(tx, e);
+                return true;
+              });
+              insertBatchData(offset + currentData.length);
+            }
+          };
+          insertBatchData(0);
         }
-        values.push(_idx);
-        if (incremental) {
-          updateRecord(tx, model, values, item.active);
-        } else if (OB.UTIL.isNullOrUndefined(item.active) || (item.active && item.active === true)) {
-          tx.executeSql(insertStatement, values, null, function (tx, e) {
-            handleError(tx, e);
-            return true;
-          });
+      }, error, function () {
+        if (data.length > 0) {
+          OB.Cache.resetCacheForModel(model.prototype.modelName);
         }
-        _idx++;
+        if (_.isFunction(success)) {
+          success();
+        }
       });
-    }, error, function () {
-      if (data.length > 0) {
-        OB.Cache.resetCacheForModel(model.prototype.modelName);
-      }
+    } else {
       if (_.isFunction(success)) {
         success();
       }
-    });
+    }
   };
 
   /**
@@ -1468,7 +1529,7 @@
       }
     }
 
-    function forceLocalDatabaseLoad(models, idx, callback) {
+    function forceLocalDatabaseLoad(models, idx, callback, tx) {
       if (idx === models.length) {
         if (callback) {
           callback();
@@ -1476,12 +1537,12 @@
         return;
       }
 
-      OB.Dal.find(models[idx], {
+      OB.Dal.findInTransaction(tx, models[idx], {
         _limit: 100
       }, function () {
-        forceLocalDatabaseLoad(models, idx + 1, callback);
+        forceLocalDatabaseLoad(models, idx + 1, callback, tx);
       }, function () {
-        forceLocalDatabaseLoad(models, idx + 1, callback);
+        forceLocalDatabaseLoad(models, idx + 1, callback, tx);
       });
     }
 
@@ -1504,8 +1565,10 @@
             OB.UTIL.localStorage.setItem('POSLastTotalRefresh', new Date().getTime());
           }
           OB.UTIL.showLoadingMessage(OB.I18N.getLabel('OBMOBC_ForcingRefreshOfLocalDatabase'));
-          forceLocalDatabaseLoad(modelsInLocalDb, 0, function () {
-            dalLoadModelsCallback();
+          OB.Data.localDB.transaction(function (tx) {
+            forceLocalDatabaseLoad(modelsInLocalDb, 0, function () {
+              dalLoadModelsCallback();
+            }, tx);
           });
         } else {
           // We know that the load of one or more models have failed.

diff --git a/web/org.openbravo.mobile.core/source/data/ob-datasource.js b/web/org.openbravo.mobile.core/source/data/ob-datasource.js
--- a/web/org.openbravo.mobile.core/source/data/ob-datasource.js
+++ b/web/org.openbravo.mobile.core/source/data/ob-datasource.js
@@ -537,6 +537,43 @@
       } else {
         OB.UTIL.showLoadingMessage(OB.I18N.getLabel('OBMOBC_LoadingMessageModelPage', [me.request.model.prototype.modelName, offset / limit + 1]));
       }
+
+      function dropModelIndexes(callback) {
+        if (typeof me.request.model.getIndexes === 'function') {
+          OB.Data.localDB.transaction(function (tx) {
+            _.forEach(me.request.model.getIndexes(), function (index) {
+              var st = "DROP INDEX IF EXISTS " + index.name + ";";
+              tx.executeSql(st, [], null, null);
+            });
+          }, function (tx, err) {
+            console.error(err);
+            callback();
+          }, function () {
+            callback();
+          });
+        } else {
+          callback();
+        }
+      }
+
+      function createModelIndexes(callback) {
+        if (typeof me.request.model.getIndexes === 'function') {
+          OB.Data.localDB.transaction(function (tx) {
+            _.forEach(me.request.model.getIndexes(), function (index) {
+              var st = me.request.model.getCreateIndexStatement(index);
+              tx.executeSql(st, [], null, null);
+            });
+          }, function (tx, err) {
+            console.error(err);
+            callback();
+          }, function () {
+            callback();
+          });
+        } else {
+          callback();
+        }
+      }
+
       me.request.exec(params, function (data, message, lastUpdated, totalRows) {
         var newLastId;
 
@@ -570,7 +607,9 @@
                 OB.UTIL.localStorage.setItem('lastUpdatedTimestamp' + me.request.model.prototype.modelName, OB.UTIL.localStorage.getItem('requestTimestamp' + me.request.model.prototype.modelName));
               }
             }
-            me.trigger('ready');
+            createModelIndexes(function () {
+              me.trigger('ready');
+            });
           }
         }
         if (data.exception) {
@@ -583,7 +622,13 @@
         }
         if (me.request.model && !me.request.model.prototype.online) {
           if (offset === 0) {
-            OB.Dal.initCache(me.request.model, data, success, handleError, incremental);
+            if (!incremental) {
+              dropModelIndexes(function () {
+                OB.Dal.initCache(me.request.model, data, success, handleError, incremental);
+              });
+            } else {
+              OB.Dal.initCache(me.request.model, data, success, handleError, incremental);
+            }
           } else {
             OB.Dal.insertData(me.request.model, data, success, handleError, incremental, dataLoaded);
           }