Merge pull request imgbot#1245 from imgbot/update_marketplace_sync

Update marketplace sync
asleekgeek · Oct 10, 2022 · fb9022a · fb9022a
2 parents b2d0d3c + cd5b1c0
commit fb9022a
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 5 deletions.
diff --git a/CompressImagesFunction/host.json b/CompressImagesFunction/host.json
@@ -8,7 +8,7 @@
    "queues": {
      "batchSize": 2,
      "maxDequeueCount" : 2,
-     "newBatchThreshold" : 1
+     "newBatchThreshold" : 2
    }
  }
 }
diff --git a/MarketplaceSyncFunction/MarketplaceSync.cs b/MarketplaceSyncFunction/MarketplaceSync.cs
@@ -1,10 +1,12 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Threading.Tasks;
 using Common;
+using Common.TableModels;
 using Install;
 using Microsoft.Azure.WebJobs;
 using Microsoft.Extensions.Logging;
@@ -45,10 +47,12 @@ public static async Task RunAsync(
                     AppId = KnownGitHubs.AppId,
                 },
                 KnownEnvironmentVariables.APP_PRIVATE_KEY);
-            var currentPlans = KnownGitHubs.Plans.Keys.Where(k => KnownGitHubs.Plans[k] == -1 || KnownGitHubs.Plans[k] >= KnownGitHubs.SmallestLimitPaidPlan);
+            var currentPlans = KnownGitHubs.Plans.Keys.Where(k =>
+                KnownGitHubs.Plans[k] == -1 || KnownGitHubs.Plans[k] >= KnownGitHubs.SmallestLimitPaidPlan);
 
             foreach (var planId in currentPlans)
             {
+                // TODO this will need to be updated to get more than 100 purchases per plan (loop with page) when we get more of the new plans bought
                 var planRequest = new HttpRequestMessage(HttpMethod.Get, $"https://api.github.com/marketplace_listing/plans/{planId}/accounts?per_page=100");
                 planRequest.Headers.Authorization = new AuthenticationHeaderValue("Bearer", jwt);
                 planRequest.Headers.Add("User-Agent", "ImgBot");
@@ -69,6 +73,9 @@ public static async Task RunAsync(
 
                 logger.LogInformation("MarketplaceSync found {NumPlans} for {PlanId}", accountsInPlan.Length, planId);
 
+                // loop through the plans and get all purchases from github in order to sync with our database
+                // ie. sync all purchases that are on git and not saved on our side
+                Dictionary<int, bool> gitData = new Dictionary<int, bool>();
                 foreach (var account in accountsInPlan)
                 {
                     var row = new Common.TableModels.Marketplace(account.id, account.login)
@@ -77,10 +84,45 @@ public static async Task RunAsync(
                         PlanId = account.marketplace_purchase.plan.id
                     };
                     await marketplaceTable.ExecuteAsync(TableOperation.InsertOrMerge(row));
+
+                    // create a map to easily check the plans at the next step
+                    gitData.Add(account.id, true);
                 }
-            }
 
-            logger.LogInformation("MarketplaceSync finished");
+                // get all purchases for a plan from our database
+                // then get all purchases for the plan from the github api
+                // remove anything that is in our database and not on the github api response
+                var query = new TableQuery<Common.TableModels.Marketplace>().Where(
+                    "PlanId eq " + planId.ToString()).Take(1000);
+
+                TableContinuationToken contToken = null;
+
+                var deletedPurchases = new List<string>();
+                do
+                {
+                    var rows = await marketplaceTable.ExecuteQuerySegmentedAsync(query, contToken);
+                    contToken = rows.ContinuationToken;
+                    if (!rows.Any())
+                        continue;
+
+                    bool redundantVariable;
+                    foreach (var purchase in rows)
+                    {
+                        if (gitData.TryGetValue(purchase.AccountId, out redundantVariable))
+                        {
+                            continue;
+                        }
+
+                        deletedPurchases.Add(purchase.PartitionKey);
+
+                        await marketplaceTable.DropRow(purchase.PartitionKey, purchase.AccountLogin);
+                    }
+                }
+                while (contToken != null);
+
+                logger.LogInformation("MarketplaceSync missing git purchases " + JsonConvert.SerializeObject(deletedPurchases));
+                logger.LogInformation("MarketplaceSync finished");
+            }
         }
     }
-}
+}