Add jargon service

Jargon service translates search strings for the market
using community-sourced abbreviations. It also creates the
jargon.yaml file in the user's save directory, for
future customization (either manual, or via the UI) and
provides the code hooks to load and update it. The
JargonLoader is capable of hot-loading new jargon.yaml
configurations without restarting Pyfa.

requirements.txt

@@ -7,4 +7,5 @@ sqlalchemy >= 1.0.5
 markdown2
 packaging
 roman
-beautifulsoup4
+beautifulsoup4
+PyYAML

service/jargon/__init__.py

@@ -0,0 +1,21 @@
+# =============================================================================
+# Copyright (C) 2018 Filip Sufitchi
+#
+# This file is part of pyfa.
+#
+# pyfa is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyfa is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyfa.  If not, see <http://www.gnu.org/licenses/>.
+# =============================================================================
+
+from .jargon import Jargon
+from .loader import JargonLoader

service/jargon/defaults.yaml

@@ -0,0 +1,3 @@
+2: II
+haml: Heavy Assault Missile Launcher
+mwd: Microwarpdrive

service/jargon/header.yaml

@@ -0,0 +1,14 @@
+# This is the default Pyfa jargon file.
+#
+# It is essentially a giant set of find/replace statements in order to translate
+# abbreviated Eve community terms into more useful full terms. It is intended
+# for translation of strings such as "haml 2" "into "Heavy Assault Missile Launcher II"..
+#
+# These abbreviations are not case-sensitive. If abbreviations collide, the
+# later one is used.
+#
+# Abbreviations with spaces are not supported.
+#
+# Syntax:
+#
+#   abbreviation: full name

service/jargon/jargon.py

@@ -0,0 +1,47 @@
+# =============================================================================
+# Copyright (C) 2018 Filip Sufitchi
+#
+# This file is part of pyfa.
+#
+# pyfa is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyfa is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyfa.  If not, see <http://www.gnu.org/licenses/>.
+# =============================================================================
+
+import config
+import pkg_resources
+
+class Jargon(object):
+    def __init__(self, rawdata: dict):
+        self._rawdata = rawdata
+
+        # copy the data to lowercase keys, ignore blank keys
+        self._data = {str(k).lower():v for k,v in rawdata.items() if k}
+
+    def get(self, term: str) -> str:
+        return self._data.get(term.lower())
+
+    def get_rawdata() -> dict:
+        return self._rawdata
+
+    def apply(self, query):
+        query_words = query.split()
+        parts = []
+
+        for word in query_words:
+            replacement = self.get(word)
+            if replacement:
+                parts.append(replacement)
+            else:
+                parts.append(word)
+
+        return ' '.join(parts)

service/jargon/loader.py

@@ -0,0 +1,81 @@
+# =============================================================================
+# Copyright (C) 2018 Filip Sufitchi
+#
+# This file is part of pyfa.
+#
+# pyfa is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyfa is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyfa.  If not, see <http://www.gnu.org/licenses/>.
+# =============================================================================
+
+import os
+import config
+import yaml
+
+from .jargon import Jargon
+from .resources import DEFAULT_DATA, DEFAULT_HEADER
+
+JARGON_PATH = os.path.join(config.savePath, 'jargon.yaml')
+
+class JargonLoader(object):
+    def __init__(self, jargon_path: str):
+        self.jargon_path = jargon_path
+        self._jargon_mtime = 0 # type: int
+        self._jargon = None # type: Jargon
+
+    def save_jargon(self, data: Jargon):
+        rawdata = data.get_rawdata()
+        with open(JARGON_PATH, 'w') as f:
+            yaml.dump(rawdata, stream=f, default_flow_style=False)
+
+    def get_jargon(self) -> Jargon:
+        if self._is_stale():
+            self._load_jargon()
+        return self._jargon
+
+    def _is_stale(self):
+        return (not self._jargon or not self._jargon_mtime or
+                self.jargon_mtime != self._get_jargon_file_mtime())
+
+    def _load_jargon(self):
+        with open(JARGON_PATH) as f:
+            rawdata = yaml.load(f)
+        self.jargon_mtime = self._get_jargon_file_mtime()
+        self._jargon = Jargon(rawdata)
+
+    def _get_jargon_file_mtime(self) -> int:
+        if not os.path.exists(self.jargon_path):
+            return 0
+        return os.stat(self.jargon_path).st_mtime
+
+    @staticmethod
+    def init_user_jargon(jargon_path):
+        values = yaml.load(DEFAULT_DATA)
+        if os.path.exists(jargon_path):
+            with open(jargon_path) as f:
+                custom_values = yaml.load(f)
+            if custom_values:
+                values.update(custom_values)
+        with open(jargon_path, 'w') as f:
+            f.write(DEFAULT_HEADER)
+            f.write('\n\n')
+            yaml.dump(values, stream=f, default_flow_style=False)
+
+    _instance = None
+    @staticmethod
+    def instance(jargon_path=None):
+        if not JargonLoader._instance:
+            jargon_path = jargon_path or JARGON_PATH
+            JargonLoader._instance = JargonLoader(jargon_path)
+        return JargonLoader._instance
+
+JargonLoader.init_user_jargon(JARGON_PATH)

service/jargon/resources.py

@@ -0,0 +1,23 @@
+# =============================================================================
+# Copyright (C) 2018 Filip Sufitchi
+#
+# This file is part of pyfa.
+#
+# pyfa is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyfa is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyfa.  If not, see <http://www.gnu.org/licenses/>.
+# =============================================================================
+
+import pkg_resources
+
+DEFAULT_DATA = pkg_resources.resource_string(__name__, 'defaults.yaml').decode()
+DEFAULT_HEADER = pkg_resources.resource_string(__name__, 'header.yaml').decode()

service/market.py

@@ -30,6 +30,7 @@
 import eos.db
 from service import conversions
 from service.settings import SettingsProvider
+from service.jargon import JargonLoader
 
 from eos.gamedata import Category as types_Category, Group as types_Group, Item as types_Item, MarketGroup as types_MarketGroup, \
     MetaGroup as types_MetaGroup, MetaType as types_MetaType
@@ -40,7 +41,6 @@
 # Event which tells threads dependent on Market that it's initialized
 mktRdy = threading.Event()
 
-
 class ShipBrowserWorkerThread(threading.Thread):
     def __init__(self):
         threading.Thread.__init__(self)
@@ -83,7 +83,9 @@ class SearchWorkerThread(threading.Thread):
     def __init__(self):
         threading.Thread.__init__(self)
         self.name = "SearchWorker"
-        pyfalog.debug("Initialize SearchWorkerThread.")
+        self.jargonLoader = JargonLoader.instance()
+        self.jargonLoader.get_jargon() # load the jargon while in an out-of-thread context, to spot any problems
+        self.jargonLoader.get_jargon().apply('foobar baz')
 
     def run(self):
         self.cv = threading.Condition()
@@ -110,13 +112,24 @@ def processSearches(self):
             else:
                 filter_ = None
 
+
             results = eos.db.searchItems(request, where=filter_,
                                          join=(types_Item.group, types_Group.category),
                                          eager=("icon", "group.category", "metaGroup", "metaGroup.parent"))
 
+            try:
+                jargon_request = self.jargonLoader.get_jargon().apply(request)
+                jargon_results = eos.db.searchItems(jargon_request, where=filter_,
+                                             join=(types_Item.group, types_Group.category),
+                                             eager=("icon", "group.category", "metaGroup", "metaGroup.parent"))
+            except Exception as e:
+                import sys
+                print(e, file=sys.stderr)
+
+
             items = set()
             # Return only published items, consult with Market service this time
-            for item in results:
+            for item in [*results, *jargon_results]:
                 if sMkt.getPublicityByItem(item):
                     items.add(item)
             wx.CallAfter(callback, items)