-
-
Notifications
You must be signed in to change notification settings - Fork 26
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merged main, bumped to latest globe view version, added some pitch/ro…
…tate controls back for globe view and adjusted minZoom.
- Loading branch information
Showing
17 changed files
with
491 additions
and
82 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
130 changes: 130 additions & 0 deletions
130
name_server/data_processing/aggregate_name_suggestions.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,130 @@ | ||
| const dotenv = require("dotenv"); | ||
| const path = require("path"); | ||
| const assert = require("assert"); | ||
| const MongoClient = require("mongodb").MongoClient; | ||
|
|
||
| const fs = require("fs"); | ||
| const csv = require("fast-csv"); | ||
|
|
||
| dotenv.config({ path: path.join(__dirname, "../.env") }); | ||
|
|
||
| const unnamedFeatureCounts = [ | ||
| { | ||
| $group: { | ||
| _id: "$levelpathid", | ||
| count: { | ||
| $sum: 1, | ||
| }, | ||
| route_url: { | ||
| $first: "$route_url", | ||
| }, | ||
| }, | ||
| }, | ||
| { | ||
| $sort: { | ||
| count: -1, | ||
| }, | ||
| }, | ||
| ]; | ||
|
|
||
| const suggestionCounts = [ | ||
| { | ||
| $group: { | ||
| _id: "$nameid", | ||
| count: { | ||
| $sum: 1, | ||
| }, | ||
| route_url: { | ||
| $push: "$route_url", | ||
| }, | ||
| suggestions: { | ||
| $push: "$suggested_name", | ||
| }, | ||
| }, | ||
| }, | ||
| { | ||
| $sort: { | ||
| count: -1, | ||
| }, | ||
| }, | ||
| ]; | ||
|
|
||
| const getExistingOverrides = async (filepath) => { | ||
| let rows = []; | ||
|
|
||
| return new Promise((resolve) => { | ||
| fs.createReadStream(path.join(__dirname, filepath)) | ||
| .pipe(csv.parse({ headers: true })) | ||
| .on("error", (error) => console.error(error)) | ||
| .on("data", (row) => rows.push(row)) | ||
| .on("end", () => resolve(rows)); | ||
| }); | ||
| }; | ||
|
|
||
| const runAggregation = async (aggregationPipeline, collection) => { | ||
| return new Promise((resolve) => { | ||
| MongoClient.connect( | ||
| process.env.MONGODB_URL, | ||
| { useNewUrlParser: true, useUnifiedTopology: true }, | ||
| function (connectErr, client) { | ||
| assert.equal(null, connectErr); | ||
|
|
||
| const coll = client.db("river_runner").collection(collection); | ||
|
|
||
| coll.aggregate(aggregationPipeline, { allowDiskUse: true, cursor: { batchSize: 1000 } }, async (cmdErr, result) => { | ||
| assert.equal(null, cmdErr); | ||
|
|
||
| const aggregationData = await result.toArray(); | ||
| client.close(); | ||
|
|
||
| resolve(aggregationData); | ||
| }); | ||
| } | ||
| ); | ||
| }); | ||
| }; | ||
|
|
||
| const main = async () => { | ||
| const csvFilepath = "../../public/data/name_overrides.csv"; | ||
|
|
||
| const existingOverrides = await getExistingOverrides(csvFilepath); | ||
| // const groupedOccurences = await runAggregation(unnamedFeatureCounts, "unnamed_features"); | ||
| const groupedSuggestions = await runAggregation(suggestionCounts, "suggestions"); | ||
|
|
||
| const overridenIDs = existingOverrides.map((d) => Number(d.levelpathid)); | ||
| const unhandledSuggestions = groupedSuggestions | ||
| .filter((d) => d.count > 3) | ||
| .filter((d) => !overridenIDs.includes(d._id)); | ||
|
|
||
| console.log(unhandledSuggestions); | ||
| // const joinedData = groupedOccurences.map((d) => { | ||
| // const suggestions = groupedSuggestions.find((a) => a._id === d._id) || { | ||
| // count: 0, | ||
| // route_url: [], | ||
| // suggestions: null, | ||
| // }; | ||
|
|
||
| // const override = existingOverrides.find((a) => Number(a.levelpathid) === d._id); | ||
|
|
||
| // return { | ||
| // levelpathid: d._id, | ||
| // sample_route_url: [...suggestions.route_url, d.route_url].filter((d) => d)[0], | ||
| // num_suggestions: suggestions.count, | ||
| // num_occurences: d.count, | ||
| // suggestions: suggestions.suggestions, | ||
| // overriden: override ? true : false, | ||
| // overriden_val: override ? override.feature_name : null, | ||
| // }; | ||
| // }); | ||
|
|
||
| // const outputFile = fs.createWriteStream("data/aggregated_suggestions.csv"); | ||
|
|
||
| // csv | ||
| // .write(joinedData, { headers: true }) | ||
| // .on("finish", function () { | ||
| // console.log("Write to CSV successfully!"); | ||
| // }) | ||
| // .pipe(outputFile); | ||
| }; | ||
|
|
||
| main(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| const mongoose = require("mongoose"); | ||
|
|
||
| const unamedFeatureSchema = new mongoose.Schema( | ||
| { | ||
| levelpathid: { | ||
| type: Number, | ||
| required: true, | ||
| }, | ||
| current_name: { | ||
| type: String, | ||
| required: true, | ||
| }, | ||
| timestamp: Number, | ||
| route_start: String, | ||
| route_url: String, | ||
| }, | ||
| { collection: "unnamed_features" } | ||
| ); | ||
|
|
||
| const UnnamedFeature = mongoose.model("UnnamedFeature", unamedFeatureSchema); | ||
| module.exports = UnnamedFeature; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
Large diffs are not rendered by default.
Oops, something went wrong.
Oops, something went wrong.