Develop

api/src/api.js

@@ -37,7 +37,8 @@ const getDataset = function (map, id, getHistoric) {
 
 function sendRows(res, rows, nextCursor) {
   const parser = new BigQueryParser(config.sourceMapping);
-  const resObj = parser.formatRowsToJson(rows);
+  const dedupe = parser.deduplicateData(rows);
+  const resObj = parser.formatRowsToJson(dedupe);
   return res.status(200).json({nextCursor, size: resObj.length, data: resObj});
 }
 

api/src/helperClasses.js

@@ -20,26 +20,32 @@ class BigQueryParser {
   constructor(mapping){
     this.mapping = mapping;
   }
+// this will only ever iterate over a max of 1000 rows, so it won't tank endpoint speed
+// like totally reformatting the query to deduplicate all the data will
+  deduplicateData (rows) {
+    const set = new Set(rows);
+    return [...set];
+  }
 
   formatRowsToJson (rows) {
     let resObj = rows.map((row) => {
+      let metadata= {};
+      row.metadata.forEach(kvp => { //only store most recent value for key
+          metadata[kvp.key] = kvp.val
+      });
+
       return {
         attribution_url: row.attribution_id,
         app_id: row.app_package_name,
         ordered_id: row.event_timestamp,
         user: {
-          id: row.user_pseudo_id,
+          id: row.uuid? row.uuid : null,
           metadata: {
-            continent: row.geo.continent,
-            country: row.geo.country,
-            region: row.geo.region,
-            city: row.geo.city,
+            ...metadata,
+            ...row.geo,
           },
           ad_attribution: {
-            source: this.getSource(row.attribution_id),
-            data: {
-              advertising_id: row.device.advertising_id,
-            },
+            source: this.getSource(row.attribution_id)
           },
         },
         event: {
@@ -49,7 +55,7 @@ class BigQueryParser {
           value_type: this.getValueType(row.label),
           value: this.getValue(row.label) || row.val,
           level: this.getLevel(row.screen) || this.getLevel(row.label)||this.getLevel(row.action)||"0",
-          profile: this.getProfile(row.screen) || 'unknown',
+          profile: row.profile? row.profile: this.getProfile(row.screen),
           rawData: {
             action: row.action,
             label: row.label,
@@ -86,7 +92,7 @@ class BigQueryParser {
     try {
       return screen.split(':')[1].trim();
     } catch(e) {
-      return null;
+      return "unknown";
     }
   }
 

api/src/sql/fetch_latest.sql

@@ -10,6 +10,31 @@ AS(
     END
 );
 WITH
+  UUIDS AS (
+    SELECT
+      user_pseudo_id,
+      params.value.string_value as uuid,
+      event_params
+    FROM
+      `{{dataset}}.events_*`,
+      UNNEST(event_params) as params
+    WHERE
+      _TABLE_SUFFIX BETWEEN '20211201' AND FORMAT_DATE('%Y%m%d', DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY))
+      AND app_info.id = @pkg_id
+      AND event_name = "LogUserUUID"
+      AND params.key = "uuid"
+  ),
+
+  UUID_AND_PROFILE AS (
+    SELECT
+      * EXCEPT(event_params),
+      params.value.int_value as profile
+    FROM
+      UUIDs,
+      UNNEST(event_params) as params
+    WHERE
+      params.key = "profile"
+  ),
   APP_INITIALIZED AS (
   SELECT
     params.value.string_value as attribution_id,
@@ -38,6 +63,13 @@ WITH
     WHERE
       params.key = 'screen'
   ),
+  METADATA AS( 
+    SELECT
+      params.*
+    FROM
+      `{{dataset}}.deep_link_parameters*` as params
+    INNER JOIN UUID_AND_PROFILE on UUID_AND_PROFILE.use_pseudo_id = params.user_pseudo_id
+  ),
   DEFAULT_SCREEN AS (
     SELECT
         attribution_id,
@@ -57,6 +89,7 @@ WITH
           WHERE params.key = 'firebase_screen'
         )
 ),
+
   LITERACY_DATA AS (
   SELECT
     *
@@ -72,6 +105,7 @@ WITH
       OR event_name = 'GamePlay')
     AND (device.advertising_id = @user_id OR user_pseudo_id = @user_id OR @user_id = '')
     ),
+
 FILTERED_LIT_DATA AS (
     SELECT
         `{{dataset}}.getValue`(params.value) as action,
@@ -82,7 +116,9 @@ FILTERED_LIT_DATA AS (
     WHERE
         params.key = "action"
         AND (params.value.string_value LIKE CONCAT(@event, '%') OR @event = '')
-), SCREENS AS (
+),
+
+ SCREENS AS (
   SELECT
     `{{dataset}}.getValue`(params.value) as screen,
     action,
@@ -102,6 +138,7 @@ FILTERED_LIT_DATA AS (
     UNNEST(SCREENS.event_params) as params
     WHERE params.key = "label"
 ),
+
 VALS AS (
     SELECT
         action,
@@ -113,26 +150,36 @@ VALS AS (
     UNNEST(LABELS.event_params) as params
     WHERE params.key = "value"
 )
+
+
 SELECT
   APP_INITIALIZED.attribution_id,
   VALS.action,
   VALS.label,
   VALS.val,
   VALS.screen,
+  METADATA.metadata
+  UUID_AND_PROFILE.profile,
+  UUID_AND_PROFILE.uuid,
   VALS.* EXCEPT(action, label, val, screen, event_params, user_properties)
 FROM
   VALS
 INNER JOIN APP_INITIALIZED on APP_INITIALIZED.user_pseudo_id = VALS.user_pseudo_id
+INNER JOIN UUID_AND_PROFILE on UUID_AND_PROFILE.user_pseudo_id = VALS.user_pseudo_id AND (VALS.screen = "Splash Screen" OR VALS.screen LIKE CONCAT("%- Profile ", UUID_AND_PROFILE.profile) OR VALS.screen LIKE CONCAT("%Profile: ", UUID_AND_PROFILE.profile))
+INNER JOIN METADATA on METADATA.user_pseudo_id = VALS.user_pseudo_id
 UNION ALL(
   SELECT
     attribution_id,
     action,
     label,
     val,
     screen,
-    * EXCEPT(attribution_id, action, screen, label, val)
+    UUID_AND_PROFILE.profile,
+    UUID_AND_PROFILE.uuid,
+    INIT_SCREEN.* EXCEPT(attribution_id, action, screen, label, val)
   FROM
     INIT_SCREEN
+    INNER JOIN UUID_AND_PROFILE on UUID_AND_PROFILE.user_pseudo_id = INIT_SCREEN.user_pseudo_id AND (INIT_SCREEN.screen = "Splash Screen" OR INIT_SCREEN.screen LIKE CONCAT("%- Profile ", UUID_AND_PROFILE.profile) OR INIT_SCREEN.screen LIKE CONCAT("%Profile: ", UUID_AND_PROFILE.profile))
 )
 UNION ALL (
   SELECT
@@ -141,8 +188,11 @@ UNION ALL (
     label,
     val,
     screen,
-    * EXCEPT(attribution_id, action, screen, label, val)
+    UUID_AND_PROFILE.profile,
+    UUID_AND_PROFILE.uuid,
+    DEFAULT_SCREEN.* EXCEPT(attribution_id, action, screen, label, val)
   FROM
     DEFAULT_SCREEN
+    INNER JOIN UUID_AND_PROFILE on UUID_AND_PROFILE.user_pseudo_id = DEFAULT_SCREEN.user_pseudo_id AND (DEFAULT_SCREEN.screen = "Splash Screen" OR DEFAULT_SCREEN.screen LIKE CONCAT("%- Profile ", UUID_AND_PROFILE.profile) OR DEFAULT_SCREEN.screen LIKE CONCAT("%Profile: ", UUID_AND_PROFILE.profile))
 )
 ORDER BY event_timestamp DESC

api/test/api.spec.js

@@ -168,18 +168,17 @@ describe('Literacy API Routes', () => {
         app_id: row.app_package_name,
         ordered_id: row.event_timestamp,
         user: {
-          id: row.user_pseudo_id,
+          id: row.uuid,
           metadata: {
+            key1: row.metadata[0].val,
+            key2: row.metadata[1].val,
             continent: row.geo.continent,
             country: row.geo.country,
             region: row.geo.region,
             city: row.geo.city,
           },
           ad_attribution: {
             source: 'no-source',
-            data: {
-              advertising_id: row.device.advertising_id,
-            },
           },
         },
         event: {
@@ -253,18 +252,17 @@ describe('Literacy API Routes', () => {
         app_id: row.app_package_name,
         ordered_id: row.event_timestamp,
         user: {
-          id: row.user_pseudo_id,
+          id: row.uuid,
           metadata: {
+            key1: row.metadata[0].val,
+            key2: row.metadata[1].val,
             continent: row.geo.continent,
             country: row.geo.country,
             region: row.geo.region,
             city: row.geo.city,
           },
           ad_attribution: {
             source: 'no-source',
-            data: {
-              advertising_id: row.device.advertising_id,
-            },
           },
         },
         event: {
@@ -295,7 +293,6 @@ describe('Literacy API Routes', () => {
       .expect(200)
       .end((err, res)=> {
         if (err) return done(err);
-        console.log("beep");
         request
           .get('/fetch_latest')
           .query({
@@ -307,7 +304,6 @@ describe('Literacy API Routes', () => {
           .expect(200)
           .end((err, fi) => {
             if (err) return done(err);
-            console.log("boop");
             fi.body.data.should.deep.equal(expected);
             done();
           });
@@ -321,18 +317,17 @@ describe('Literacy API Routes', () => {
         app_id: row.app_package_name,
         ordered_id: row.event_timestamp,
         user: {
-          id: row.user_pseudo_id,
+          id: row.uuid,
           metadata: {
+            key1: row.metadata[0].val,
+            key2: row.metadata[1].val,
             continent: row.geo.continent,
             country: row.geo.country,
             region: row.geo.region,
             city: row.geo.city,
           },
           ad_attribution: {
             source: 'no-source',
-            data: {
-              advertising_id: row.device.advertising_id,
-            },
           },
         },
         event: {
@@ -364,7 +359,6 @@ describe('Literacy API Routes', () => {
       .end((err, res)=> {
         if (err) return done(err);
 
-        console.log(`cursor: ${res.body.nextCursor}`)
         request
           .get('/fetch_latest')
           .query({

api/test/bigQueryParser.spec.js

@@ -96,16 +96,16 @@ describe('BigQueryHelper', () => {
       actual.should.equal(expected.event.profile);
     });
 
-    it('should return null on an improperly formatted string', () => {
+    it('should return "unknown" on an improperly formatted string', () => {
       const row = fixtures[6].row;
       const actual = bqParser.getProfile(row.screen);
-      expect(actual).to.be.null;
+      expect(actual).to.equal("unknown");
     });
 
-    it('should return null on an improperly formatted string', () => {
+    it('should return "unknown" on an improperly formatted string', () => {
       const row = fixtures[2].row;
       const actual = bqParser.getProfile(row.screen);
-      expect(actual).to.be.null;
+      expect(actual).to.equal("unknown");
     });
   });
 
@@ -195,30 +195,24 @@ describe('BigQueryHelper', () => {
     });
   });
 
+  describe('deduplicateData', () => {
+    it('should remove duplicate rows', () => {
+      const rows = fixtures.map((row) => {return row.row});
+      const data = rows.concat(rows);
+      const result = bqParser.deduplicateData(data);
+      result.should.deep.equal(rows);
+    });
+    it('should not change an array of unique objects', () => {
+      const rows = fixtures.map((row) => {return row.row});
+      const result = bqParser.deduplicateData(rows);
+      result.should.deep.equal(rows);
+    });
+  });
+
   describe('formatRowsToJson', () => {
     it('should parse data correctly', () => {
-      const rows = [
-        fixtures[0].row,
-        fixtures[1].row,
-        fixtures[2].row,
-        fixtures[3].row,
-        fixtures[4].row,
-        fixtures[5].row,
-        fixtures[6].row,
-        fixtures[7].row,
-        fixtures[8].row
-      ];
-      const expected = [
-        fixtures[0].expected,
-        fixtures[1].expected,
-        fixtures[2].expected,
-        fixtures[3].expected,
-        fixtures[4].expected,
-        fixtures[5].expected,
-        fixtures[6].expected,
-        fixtures[7].expected,
-        fixtures[8].expected
-      ];
+      const rows = fixtures.map((row) => {return row.row});
+      const expected = fixtures.map((row) => {return row.expected});
       const actual = bqParser.formatRowsToJson(rows);
       actual.should.deep.equal(expected);
     });