0034669: ProcessHQLQuery returns duplicates causing errors when inserting data and really slow login with many products

diff --git a/src-db/database/sourcedata/AD_REF_LIST.xml b/src-db/database/sourcedata/AD_REF_LIST.xml
--- a/src-db/database/sourcedata/AD_REF_LIST.xml
+++ b/src-db/database/sourcedata/AD_REF_LIST.xml
@@ -125,6 +125,18 @@
 <!--45604629CD6144F7BB4AA45EFE87C258-->  <AD_MODULE_ID><![CDATA[08943B85ADF64E708797A753E5B6AAEE]]></AD_MODULE_ID>
 <!--45604629CD6144F7BB4AA45EFE87C258--></AD_REF_LIST>
 
+<!--461A2CE20C044EF9A82377799F586066--><AD_REF_LIST>
+<!--461A2CE20C044EF9A82377799F586066-->  <AD_REF_LIST_ID><![CDATA[461A2CE20C044EF9A82377799F586066]]></AD_REF_LIST_ID>
+<!--461A2CE20C044EF9A82377799F586066-->  <AD_CLIENT_ID><![CDATA[0]]></AD_CLIENT_ID>
+<!--461A2CE20C044EF9A82377799F586066-->  <AD_ORG_ID><![CDATA[0]]></AD_ORG_ID>
+<!--461A2CE20C044EF9A82377799F586066-->  <ISACTIVE><![CDATA[Y]]></ISACTIVE>
+<!--461A2CE20C044EF9A82377799F586066-->  <VALUE><![CDATA[OBMOBC_MasterdataBatchSize]]></VALUE>
+<!--461A2CE20C044EF9A82377799F586066-->  <NAME><![CDATA[Masterdata models batch size ]]></NAME>
+<!--461A2CE20C044EF9A82377799F586066-->  <DESCRIPTION><![CDATA[Send request model data in batches of n records]]></DESCRIPTION>
+<!--461A2CE20C044EF9A82377799F586066-->  <AD_REFERENCE_ID><![CDATA[A26BA480E2014707B47257024C3CBFF7]]></AD_REFERENCE_ID>
+<!--461A2CE20C044EF9A82377799F586066-->  <AD_MODULE_ID><![CDATA[08943B85ADF64E708797A753E5B6AAEE]]></AD_MODULE_ID>
+<!--461A2CE20C044EF9A82377799F586066--></AD_REF_LIST>
+
 <!--47F9D6D152C9487FBDFB10929511A53A--><AD_REF_LIST>
 <!--47F9D6D152C9487FBDFB10929511A53A-->  <AD_REF_LIST_ID><![CDATA[47F9D6D152C9487FBDFB10929511A53A]]></AD_REF_LIST_ID>
 <!--47F9D6D152C9487FBDFB10929511A53A-->  <AD_CLIENT_ID><![CDATA[0]]></AD_CLIENT_ID>
diff --git a/src/org/openbravo/mobile/core/process/ProcessHQLQuery.java b/src/org/openbravo/mobile/core/process/ProcessHQLQuery.java
--- a/src/org/openbravo/mobile/core/process/ProcessHQLQuery.java
+++ b/src/org/openbravo/mobile/core/process/ProcessHQLQuery.java
@@ -103,16 +103,14 @@
       JSONArray remoteFilters = jsonsent.has("remoteFilters") && !jsonsent.isNull("remoteFilters") ? jsonsent
           .getJSONArray("remoteFilters") : null;
 
+      boolean isMasterdata = jsonsent.has("_isMasterdata") ? jsonsent.getBoolean("_isMasterdata")
+          : false;
       int totalRows = 0;
       int queryRows = 0;
       int limit = -1;
-      int offset = -1;
       if (jsonsent.has("_limit") && !jsonsent.isNull("_limit")) {
         limit = jsonsent.getInt("_limit");
       }
-      if (jsonsent.has("_offset") && !jsonsent.isNull("_offset")) {
-        offset = jsonsent.getInt("_offset");
-      }
       boolean firstQuery = true;
       Map<String, Object> parameterValues = this.getParameterValues(jsonsent);
       List<String> queries = getQuery(jsonsent);
@@ -147,8 +145,8 @@
           query.setMaxResults(limit);
         }
 
-        if (offset > -1) {
-          query.setFirstResult(offset);
+        if (jsonsent.has("_offset")) {
+          query.setFirstResult(jsonsent.getInt("_offset"));
         }
 
         List<String> queryParams = new ArrayList<String>(Arrays.asList(query.getNamedParameters()));
@@ -210,11 +208,15 @@
         if (totalRows > 0) {
           firstQuery = false;
         }
-        if (queryRows < limit) {
-          offset = 0;
-        }
-        if (limit > -1) {
-          limit = limit - totalRows;
+        /*
+         * Do not recalculate the limit for next queries when: 1.- We send an array of limits where
+         * we define the limit for each query. 2.- When the model is a masterdata model. In this
+         * case limit is used for pagination.
+         * 
+         * It must be used just when we do remote queries
+         */
+        if (limit > -1 && !isMasterdata) {
+          limit = limit - queryRows;
         }
       }
       if (!streamOpened) {
diff --git a/web/org.openbravo.mobile.core/source/data/ob-datasource.js b/web/org.openbravo.mobile.core/source/data/ob-datasource.js
--- a/web/org.openbravo.mobile.core/source/data/ob-datasource.js
+++ b/web/org.openbravo.mobile.core/source/data/ob-datasource.js
@@ -279,41 +279,38 @@
     var data = {};
 
     if (params) {
-      if (params._limit) {
-        data._limit = params._limit;
-        delete params._limit;
-      }
-      if (params._offset || params._offset === 0) {
-        data._offset = params._offset;
-        delete params._offset;
-      }
       p = {};
       for (i in params) {
         if (params.hasOwnProperty(i)) {
-          if (typeof params[i] === 'string') {
-            p[i] = {
-              value: params[i],
-              type: 'string'
-            };
-          } else if (typeof params[i] === 'number') {
-            if (params[i] === Math.round(params[i])) {
+          //Parameters starting with '_' are not query params so we will set them in data object
+          if (i.startsWith('_')) {
+            data[i] = params[i];
+          } else {
+            if (typeof params[i] === 'string') {
               p[i] = {
                 value: params[i],
-                type: 'long'
+                type: 'string'
+              };
+            } else if (typeof params[i] === 'number') {
+              if (params[i] === Math.round(params[i])) {
+                p[i] = {
+                  value: params[i],
+                  type: 'long'
+                };
+              } else {
+                p[i] = {
+                  value: params[i],
+                  type: 'bigdecimal'
+                };
+              }
+            } else if (typeof params[i] === 'boolean') {
+              p[i] = {
+                value: params[i],
+                type: 'boolean'
               };
             } else {
-              p[i] = {
-                value: params[i],
-                type: 'bigdecimal'
-              };
+              p[i] = params[i];
             }
-          } else if (typeof params[i] === 'boolean') {
-            p[i] = {
-              value: params[i],
-              type: 'boolean'
-            };
-          } else {
-            p[i] = params[i];
           }
         }
       }
@@ -434,6 +431,7 @@
       params = params || {};
       params._limit = limit;
       params._offset = offset;
+      params._isMasterdata = true;
       if (offset === 0) {
         OB.UTIL.localStorage.setItem('requestTimestamp' + me.request.model.prototype.modelName, (new Date()).getTime());
         OB.UTIL.showLoadingMessage(OB.I18N.getLabel('OBMOBC_LoadingMessageModel', [me.request.model.prototype.modelName]));
@@ -489,8 +487,7 @@
     };
 
     this.cache = null;
-
-    handleIncrementalRequest(35000, 0, params, incremental);
+    handleIncrementalRequest(OB.MobileApp.model.hasPermission('OBMOBC_MasterdataBatchSize', true) ? OB.DEC.abs(OB.MobileApp.model.hasPermission('OBMOBC_MasterdataBatchSize', true)) : 10000, 0, params, incremental);
   };
 
   OB.DS.DataSource.prototype.find = function (filter, callback) {