remove no longer maintained MOA reords

src/moa/index.js

@@ -544,6 +544,18 @@ const parseRelevance = (moaRecord) => {
     return relevance;
 };
 
+const removeRecords = async (conn, records) => {
+    if (records.length && records.length > 0) {
+        for (const record of records) {
+            try{
+                await conn.deleteRecord('Statement', record['@rid']);
+                logger.info(`Removing Statement ${record['@rid']} that are out of date`);
+            } catch (err) {
+                logger.warn(`${err}`);
+            }
+        }
+    }
+};
 
 const upload = async ({ conn, url = 'https://moalmanac.org/api/assertions' }) => {
     const source = await conn.addSource(SOURCE_DEFN);
@@ -559,6 +571,12 @@ const upload = async ({ conn, url = 'https://moalmanac.org/api/assertions' }) =>
 
     const existing = {};
 
+    const newIds = records.map(record => record.assertion_id);
+    let toRemove = existingRecords.filter(
+        record => !newIds.includes(parseInt(record.sourceId, 10)),
+    );
+    await removeRecords(conn, toRemove);
+
     for (const record of existingRecords) {
         const key = record.sourceId;
 
@@ -573,41 +591,31 @@ const upload = async ({ conn, url = 'https://moalmanac.org/api/assertions' }) =>
             logger.info(`loading: ${rawRecord.assertion_id} / ${records.length}`);
             const record = fixStringNulls(rawRecord);
 
+            if (record.assertion_id === 258){
+                logger.info(`loading: ${rawRecord.assertion_id}`);
+            }
             // handle empty space in url
             if (record.sources[0].url && record.sources[0].url.includes(' ')) {
                 record.sources[0].url = record.sources[0].url.replace(/\s/g, '');
             }
             checkSpec(validateMoaRecord, record);
             const key = `${record.assertion_id}`;
-            const lastUpdate = new Date(record.last_updated).getTime();
             const relevance = parseRelevance(record);
 
-            // do we have the expected number of GraphKB records for this MOA assertion
-            if (existing[key] && existing[key].length === relevance.length) {
-                // check the last update date of the assertion against the timestamp in GraphKB
-                if (existing[key].every(r => lastUpdate <= r.updatedAt)) {
-                    logger.debug('Skip. Current record exists and does not need updating');
-                    counts.skipped++;
-                    continue;
-                }
-            }
             const updatedRecords = (await loadRecord(conn, record, source, relevance)).map(r => r['@rid']);
 
             if (existing[key]) {
-                const toRemove = existing[key].map(r => r['@rid']).filter(r => !updatedRecords.includes(r));
-
-                if (toRemove.length) {
-                    logger.warn(`Removing ${toRemove.length} records that are out of date`);
-
-                    for (const recordId of toRemove) {
-                        await conn.deleteRecord('Statement', recordId);
-                    }
-                }
+                toRemove = existing[key].filter(r => !updatedRecords.includes(r['@rid']));
+                await removeRecords(conn, toRemove);
             }
             counts.success++;
         } catch (err) {
             logger.warn(`${err}`);
             counts.error++;
+            toRemove = existingRecords.filter(
+                record => rawRecord.assertion_id === parseInt(record.sourceId, 10),
+            );
+            await removeRecords(conn, toRemove);
 
             if (err.toString().includes('Cannot read property')) {
                 throw err;