From 5e1565b686f8de73e529c5d343f92d82b1c9547e Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 28 May 2020 23:05:12 +0200 Subject: [PATCH 01/16] Use setuptools-scm; configuration in setup.cfg --- MANIFEST.in | 3 ++ message_ix/__init__.py | 14 +++++--- setup.cfg | 38 ++++++++++++++++++++++ setup.py | 72 ++---------------------------------------- 4 files changed, 52 insertions(+), 75 deletions(-) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..51eba78bc --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +exclude .* +prune .github +prune ci diff --git a/message_ix/__init__.py b/message_ix/__init__.py index b59e5abdb..8db64a7bb 100644 --- a/message_ix/__init__.py +++ b/message_ix/__init__.py @@ -3,8 +3,9 @@ from ixmp import config from ixmp.model import MODELS +from pkg_resources import DistributionNotFound, get_distribution -from .core import Scenario # noqa: F401 +from .core import Scenario from .models import MACRO, MESSAGE, MESSAGE_MACRO @@ -17,16 +18,19 @@ 'config', ] +try: + __version__ = get_distribution(__name__).version +except DistributionNotFound: + # Package is not installed + __version__ = "999" + # Register configuration keys with ixmp core and set default config.register('message model dir', Path, Path(__file__).parent / 'model') - # Register models with ixmp core MODELS['MACRO'] = MACRO MODELS['MESSAGE'] = MESSAGE MODELS['MESSAGE-MACRO'] = MESSAGE_MACRO - -__version__ = MESSAGE.read_version() - +# Create the top-level logger log = logging.getLogger(__name__) diff --git a/setup.cfg b/setup.cfg index 1fabd17ba..29f637001 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,42 @@ +[metadata] +name = message_ix +author = IIASA Energy Program +author_email = message_ix@iiasa.ac.at +license = Apache +description = the MESSAGEix integrated assessment model +long_description_content_type = text/markdown +long_description = file:README.md +url = http://github.com/iiasa/message_ix + +[options] +packages = message_ix +zip_safe = True +include_package_data = True +install_requires = + click + ixmp >= 2.0.0 + numpy + pandas + PyYAML + xlrd +setup_requires = + setuptools >= 41 + setuptools_scm + +[options.extras_require] +docs = numpydoc; sphinx >= 3.0; sphinx_rtd_theme; sphinxcontrib-bibtex +reporting = pint; pyam-iamc +tests = asyncssh; pytest >= 5; requests +tutorial = jupyter; matplotlib; plotnine + +[options.entry_points] +console_scripts = + message-ix = message_ix.cli:main + [tool:pytest] # Disable faulthandler plugin on Windows to prevent spurious console noise addopts = --cov=message_ix --cov-config=ci/coveragerc --cov-report= -p no:faulthandler + +[aliases] +test = pytest diff --git a/setup.py b/setup.py index 13c4c9618..76755a445 100644 --- a/setup.py +++ b/setup.py @@ -1,72 +1,4 @@ #!/usr/bin/env python -import os -import re +from setuptools import setup -from setuptools import find_packages, setup - - -fname = os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'message_ix', 'model', 'version.gms') -with open(fname) as f: - s = str(f.readlines()) - -VERSION = '{}.{}.{}'.format( - re.search('VERSION_MAJOR "(.+?)"', s).group(1), - re.search('VERSION_MINOR "(.+?)"', s).group(1), - re.search('VERSION_PATCH "(.+?)"', s).group(1), -) - -with open('README.md') as f: - LONG_DESCRIPTION = f.read() - -INSTALL_REQUIRES = [ - 'click', - 'ixmp>=2.0.0', - 'pandas', - 'xlrd', - 'XlsxWriter', -] - -EXTRAS_REQUIRE = { - 'tests': ['asyncssh', 'pytest>=4.0'], - 'docs': ['numpydoc', 'sphinx>=2.3', 'sphinx_rtd_theme', - 'sphinxcontrib-bibtex'], - 'reporting': ['pyam-iamc'], - 'tutorial': ['jupyter', 'matplotlib', 'plotnine'], -} - - -def all_gams_files(path, strip=None): - paths = [] - for root, dirnames, files in os.walk(path): - for f in [_f for _f in files - if os.path.splitext(_f)[1] in ['.gms', '.opt', '.md']]: - paths.append(os.path.join(root, f)) - if strip: - n = len(strip) if strip.endswith(os.sep) else len(strip + os.sep) - paths = [x[n:] for x in paths] - return paths - - -setup( - name='message_ix', - version=VERSION, - description='The MESSAGEix Integrated Assessment Model', - author='IIASA Energy Program', - author_email='message_ix@iiasa.ac.at', - long_description=LONG_DESCRIPTION, - long_description_content_type='text/markdown', - url='http://github.com/iiasa/message_ix', - install_requires=INSTALL_REQUIRES, - extras_require=EXTRAS_REQUIRE, - packages=find_packages(), - package_dir={'message_ix': 'message_ix'}, - package_data={ - 'message_ix': all_gams_files('message_ix/model', strip='message_ix') - }, - entry_points={ - 'console_scripts': [ - 'message-ix=message_ix.cli:main' - ], - } -) +setup(use_scm_version=True) From e6f9bba23a6afd0bded7bdd0dfe7b3a2d96c8ce3 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 28 May 2020 23:08:45 +0200 Subject: [PATCH 02/16] Replace f'{foo!r}' with f'{foo}'/repr(foo) --- message_ix/reporting/pyam.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/message_ix/reporting/pyam.py b/message_ix/reporting/pyam.py index b343a2aac..597098eee 100644 --- a/message_ix/reporting/pyam.py +++ b/message_ix/reporting/pyam.py @@ -65,7 +65,9 @@ def as_pyam(scenario, quantity, replace_vars=None, year_time_dim=None, if len(df) and unit: from_unit = df['unit'].unique() if len(from_unit) > 1: - raise ValueError(f'cannot convert non-unique units {from_unit!r}') + raise ValueError( + f'cannot convert non-unique units {repr(from_unit)}' + ) q = pint.Quantity(df['value'].values, from_unit[0]).to(unit) df['value'] = q.magnitude df['unit'] = unit @@ -78,8 +80,10 @@ def as_pyam(scenario, quantity, replace_vars=None, year_time_dim=None, # Warn about extra columns extra = sorted(set(df.columns) - set(IAMC_IDX + ['year', 'time', 'value'])) if extra: - log.warning(f'Extra columns {extra!r} when converting ' - f'{quantity.name!r} to IAMC format') + log.warning( + f'Extra columns {repr(extra)} when converting ' + f'{repr(quantity.name)} to IAMC format' + ) return IamDataFrame(df) From 6f8a11b8263708bdb16c1ff198dd4aaa465e48fa Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 28 May 2020 23:24:36 +0200 Subject: [PATCH 03/16] Simplify doc/source/conf.py per sphinx-quickstart 3.0 --- doc/source/conf.py | 312 +++++++-------------------------------------- 1 file changed, 45 insertions(+), 267 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 291ae1ab8..50b93b1b1 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,36 +1,42 @@ -# -*- coding: utf-8 -*- +# Configuration file for the Sphinx documentation builder. # -# message documentation build configuration file, created by -# sphinx-quickstart on Mon Feb 8 16:15:24 2016. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. -import sys import os +from pathlib import Path +import sys + +from pkg_resources import get_distribution + +sys.path.insert(0, os.path.abspath('..')) + + +# -- Project information ----------------------------------------------------- -try: - from pathlib import Path -except ImportError: - pass +project = 'MESSAGEix' +copyright = '2020, IIASA Energy Program' +author = 'MESSAGEix Developers' -import message_ix +# The major project version, used as the replacement for |version|. +version = get_distribution("message_ix").version +# The full project version, used as the replacement for |release| and e.g. in +# the HTML templates. +release = version -# -- General configuration ------------------------------------------------ -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# -- General configuration ---------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -sys.path.append('.') extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', @@ -42,86 +48,29 @@ 'sphinxcontrib.bibtex', 'sphinx.ext.autosummary', 'sphinx.ext.napoleon', + 'grab_gams_doc' ] -# Extension only works for Python 3 -if sys.version_info[0] == 3: - sys.path.insert(0, os.path.abspath('..')) - extensions.append('grab_gams_doc') - - # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'MESSAGEix' -copyright = u'2020, IIASA, Energy Program' -author = u'MESSAGEix Developers' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -# version = as determined from function above -# The full version, including alpha/beta/rc tags. -version = message_ix.__version__ -release = message_ix.__version__ - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_themes/*'] -# The reST default role (used for this markup: `text`) to use for all -# documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True +# A string of reStructuredText that will be included at the beginning of every +# source file that is read. +rst_prolog = r""" +.. |MESSAGEix| replace:: MESSAGE\ :emphasis:`ix` -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False +.. |ixmp| replace:: :emphasis:`ix modeling platform` -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +.. role:: strike -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +.. role:: underline -# If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = True +""" # -- Options for HTML output ---------------------------------------------- @@ -129,182 +78,19 @@ # a list of builtin themes. html_theme = 'sphinx_rtd_theme' -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# A list of CSS files. +html_css_files = ["custom.css"] # The name of an image file (relative to this directory) to place at the top # of the sidebar. html_logo = '_static/logo_white.png' -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] -def setup(app): - app.add_stylesheet('custom.css') - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -#html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Language to be used for generating the HTML full-text search index. -# Sphinx supports the following languages: -# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' -# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' -#html_search_language = 'en' - -# A dictionary with options for the search language support, empty by default. -# Now only 'ja' uses this config value -#html_search_options = {'type': 'default'} - -# The name of a javascript file (relative to the configuration directory) that -# implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' - -# Output file base name for HTML help builder. -htmlhelp_basename = 'MESSAGEix_doc' - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - #'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - #'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - #'preamble': '', - - # Latex figure (float) alignment - #'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'message.tex', u'message Documentation', - u'IIASA Energy Program', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'message', u'message Documentation', - [author], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'message', u'message Documentation', - author, 'message', 'One line description of project.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False - # -- Options for sphinx.ext.extlinks ------------------------------------------ extlinks = { @@ -324,21 +110,13 @@ def setup(app): } -# prolog for all rst files -rst_prolog = r""" -.. |MESSAGEix| replace:: MESSAGE\ :emphasis:`ix` - -.. |ixmp| replace:: :emphasis:`ix modeling platform` - -.. |version| replace:: {} +# -- Options for sphinx.ext.todo ---------------------------------------------- -.. role:: strike +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True -.. role:: underline -""".format(version) +# -- Options for grab_gams_doc ------------------------------------------------ -# Configuration for grab_gams_doc extension -if sys.version_info[0] == 3: - gams_source_dir = Path(__file__).parents[2] / 'message_ix' / 'model' - gams_target_dir = 'model' +gams_source_dir = Path(__file__).parents[2] / 'message_ix' / 'model' +gams_target_dir = 'model' From f0a99db0b8aa3cde07c42c2ac7e6a85e5784e079 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 28 May 2020 23:51:51 +0200 Subject: [PATCH 04/16] Satisfy flake8 (stricter than Stickler) --- message_ix/core.py | 2 +- tutorial/Austrian_energy_system/tools.py | 11 ++++------- tutorial/utils/plotting.py | 7 +++---- tutorial/utils/run_scenarios.py | 14 +++++++++----- tutorial/westeros/tools.py | 9 +++------ 5 files changed, 20 insertions(+), 23 deletions(-) diff --git a/message_ix/core.py b/message_ix/core.py index 6507625cf..633b190a7 100755 --- a/message_ix/core.py +++ b/message_ix/core.py @@ -482,7 +482,7 @@ def rename(self, name, mapping, keep=False): try: self.check_out() commit = True - except: + except RuntimeError: commit = False keys = list(mapping.keys()) diff --git a/tutorial/Austrian_energy_system/tools.py b/tutorial/Austrian_energy_system/tools.py index 66eb2bc60..05503aac1 100644 --- a/tutorial/Austrian_energy_system/tools.py +++ b/tutorial/Austrian_energy_system/tools.py @@ -1,10 +1,7 @@ +from pathlib import Path import sys -import os -here = os.path.dirname(os.path.abspath(os.path.realpath(__file__))) +sys.path.append(str(Path(__file__).parents[1] / "utils")) -path = os.path.join(here, '..', 'utils') -sys.path.append(path) - -from plotting import * -from run_scenarios import * +from plotting import Plots # noqa: E402, F401 +from run_scenarios import * # noqa: E402, F401, F403 diff --git a/tutorial/utils/plotting.py b/tutorial/utils/plotting.py index 995163e86..74e58cb94 100644 --- a/tutorial/utils/plotting.py +++ b/tutorial/utils/plotting.py @@ -3,7 +3,6 @@ class Plots(object): - def __init__(self, ds, country, firstyear=2010, input_costs='$/GWa'): self.ds = ds self.country = country @@ -54,11 +53,11 @@ def equ_data(self, equ, value, baseyear=False, subset=None): def plot_demand(self, light_demand, elec_demand): fig, ax = plt.subplots() - l = light_demand['value'] - l.plot(ax=ax, label='light') + light = light_demand['value'] + light.plot(ax=ax, label='light') e = elec_demand['value'] e.plot(ax=ax, label='elec') - (l + e).plot(ax=ax, label='total') + (light + e).plot(ax=ax, label='total') plt.ylabel('GWa') plt.xlabel('Year') plt.legend(loc='best') diff --git a/tutorial/utils/run_scenarios.py b/tutorial/utils/run_scenarios.py index ea60c4b21..eccb387a9 100644 --- a/tutorial/utils/run_scenarios.py +++ b/tutorial/utils/run_scenarios.py @@ -1,8 +1,7 @@ +from contextlib import contextmanager import subprocess import sys -from contextlib import contextmanager - import ixmp import message_ix @@ -15,9 +14,14 @@ def check_local_model(model, notebook, shell=False): mp.close_db() pyversion = sys.version_info[0] - cmd = "jupyter nbconvert {} --ExecutePreprocessor.kernel_name='python{}' --execute" - cmd = cmd.format(notebook, pyversion) - subprocess.check_call(cmd.split(), shell=shell) + cmd = [ + "jupyter", + "nbconvert", + notebook, + f"--ExecutePreprocessor.kernel_name='python{pyversion}'", + "--execute", + ] + subprocess.check_call(cmd, shell=shell) @contextmanager diff --git a/tutorial/westeros/tools.py b/tutorial/westeros/tools.py index 31c2c9e5d..2974d50e2 100644 --- a/tutorial/westeros/tools.py +++ b/tutorial/westeros/tools.py @@ -1,9 +1,6 @@ +from pathlib import Path import sys -import os -here = os.path.dirname(os.path.abspath(os.path.realpath(__file__))) +sys.path.append(str(Path(__file__).parents[1] / "utils")) -path = os.path.join(here, '..', 'utils') -sys.path.append(path) - -from plotting import * +from plotting import Plots # noqa: E402, F401 From 393cd2a99e9482251c20f39d80d867721290cfcc Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 29 May 2020 13:54:52 +0200 Subject: [PATCH 05/16] Update comments of GDX version check --- message_ix/model/MESSAGE/version_check.gms | 19 +++++++++---- message_ix/model/version.gms | 33 +++++++++++++--------- message_ix/models.py | 16 ----------- 3 files changed, 32 insertions(+), 36 deletions(-) diff --git a/message_ix/model/MESSAGE/version_check.gms b/message_ix/model/MESSAGE/version_check.gms index e89f58498..b5749e7b6 100644 --- a/message_ix/model/MESSAGE/version_check.gms +++ b/message_ix/model/MESSAGE/version_check.gms @@ -1,7 +1,6 @@ - -*----------------------------------------------------------------------------------------------------------------------* -* load ixmp MESSAGE-scheme version number from the input gdx and check whether it matches the MESSAGEix version number * -*----------------------------------------------------------------------------------------------------------------------* +* Version check * +* +* See technical description in ../version.gms. Parameter MESSAGE_ix_version(*); @@ -10,8 +9,16 @@ $LOAD MESSAGE_IX_version $GDXIN IF ( NOT ( MESSAGE_IX_version("major") = %VERSION_MAJOR% AND MESSAGE_IX_version("minor") = %VERSION_MINOR% ), + logfile.nw = 1; + logfile.nd = 0; + put_utility 'log' / '***'; - put_utility 'log' / '*** Abort "The MESSAGEix version and the MESSAGE-scheme definition in the installed ixmp package do not match!"'; + put_utility 'log' / '*** ABORT'; + put_utility 'log' / '*** GDX file was written by an ixmp.jar incompatible with this version of MESSAGEix:'; + put_utility 'log' / '*** %in%'; + put_utility 'log' / '*** ...has version ' MESSAGE_IX_version("major") '.' MESSAGE_IX_version("minor") + ' while version.gms has %VERSION_MAJOR%.%VERSION_MINOR%'; put_utility 'log' / '***'; - abort "Incompatible versions of MESSAGEix and ixmp"; + + abort "GDX file incompatible with current version of MESSAGEix"; ) ; diff --git a/message_ix/model/version.gms b/message_ix/model/version.gms index 6e986928e..384fdcc1d 100644 --- a/message_ix/model/version.gms +++ b/message_ix/model/version.gms @@ -1,18 +1,23 @@ -* This file specifies the release version number of MESSAGEix. -* The version number must match the MESSAGEix-scheme version number -* in the compiled `ixmp.jar`. - +* GDX scheme version +* * !!! DO NOT CHANGE VALUES IN THIS FILE MANUALLY !!! - -* Changes have to be done by pulling the respective version -* from the Github repository at https://github.com/iiasa/message_ix, -* or by updating the `message_ix` package -* using `conda update -c conda-forge message-ix`. +* Instead, update message_ix and ixmp as described in the documentation. +* +* Technical details: +* +* These numbers describe the contents of the GDX file written by the Java code +* in ixmp.jar. The Java code automatically generates some contents, e.g., set +* elements, in a way that cannot be controlled or overriden by Python ixmp or +* message_ix. +* +* Formerly, these numbers were incremented in ixmp_source, ixmp.jar, and this +* file, with every release. Currently, they will be incremented if (and *only* +* if) there are changes in the behaviour of the Java code that must be synced +* with corresponding changes in the GAMS source files in this directory. +* +* Eventually, all automatic behaviour will be moved from ixmp_source (Java) to +* ixmp (Python); see https://github.com/iiasa/message_ix/issues/254. At that +* point, both this file and MESSAGE/version_check.gms can be removed. $SETGLOBAL VERSION_MAJOR "2" $SETGLOBAL VERSION_MINOR "0" -$SETGLOBAL VERSION_PATCH "0" - -* This file is imported by `message_ix/__init__.py`. -* In the documentation rst files, the tag ``|version|`` in any mark-up docstring -* is replaced by '%VERSION_MAJOR%.%VERSION_MINOR%'. diff --git a/message_ix/models.py b/message_ix/models.py index 43a6469da..4cf0f9362 100644 --- a/message_ix/models.py +++ b/message_ix/models.py @@ -72,22 +72,6 @@ class GAMSModel(ixmp.model.gams.GAMSModel): 'use_temp_dir': False, }, ixmp.model.gams.GAMSModel.defaults) - @classmethod - def read_version(cls): - """Retrieve MESSAGE version string from version.gms.""" - version_file = Path(config.get('message model dir'), 'version.gms') - if not version_file.exists(): - # Only exists here on install - version_file = Path(__file__).parent / 'model' / 'version.gms' - - s = version_file.read_text() - - return '{}.{}.{}'.format( - re.search('VERSION_MAJOR "(.+?)"', s).group(1), - re.search('VERSION_MINOR "(.+?)"', s).group(1), - re.search('VERSION_PATCH "(.+?)"', s).group(1), - ) - @classmethod def initialize(cls, scenario): """Set up *scenario* with required sets and parameters for MESSAGE. From d85ffcdacf36c52b8b98380c11e7d716f7d9f02d Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 29 May 2020 16:45:34 +0200 Subject: [PATCH 06/16] Use GitHub API to check tags in 'message-ix dl'; only extract tutorials --- message_ix/cli.py | 44 ++++++++++++++++++++++++++++-------- message_ix/tests/test_cli.py | 14 +++++++----- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/message_ix/cli.py b/message_ix/cli.py index bc431e55b..a464657c0 100644 --- a/message_ix/cli.py +++ b/message_ix/cli.py @@ -1,3 +1,4 @@ +import json from pathlib import Path from shutil import copyfile from urllib.request import urlretrieve @@ -78,29 +79,54 @@ def copy_model(path, overwrite, set_default): @click.option('--branch', help='Repository branch to download from (e.g., master).') @click.option('--tag', - help='Repository tag to download from (e.g., 1.0.0).') + help='Repository tag to download from (e.g., v1.0.0).') @click.argument('path', type=click.Path()) def dl(branch, tag, path): if tag and branch: raise click.BadOptionUsage('Can only provide one of `tag` or `branch`') - elif tag is None and branch is None: - tag = '{}'.format(message_ix.__version__) - print(tag) - zipname = '{}.zip'.format(branch or 'v' + tag) - url = 'https://github.com/iiasa/message_ix/archive/{}'.format(zipname) + if tag or branch is None: + # Get tag information using GitHub API + url = "https://api.github.com/repos/iiasa/message_ix/tags" + with open(urlretrieve(url)[0]) as f: + tags_info = json.load(f) + + if tag is None: + tag = tags_info[0]["name"] + print(f"Default: latest release {tag}") + + url = None + for info in tags_info: + if info["name"] == tag: + url = info["zipball_url"] + break + + if url is None: + raise ValueError(f"tag {repr(tag)} does not exist") + + zipname = f"{tag}.zip" + else: + # Construct URL and filename from branch + zipname = f"{branch}.zip" + url = f"https://github.com/iiasa/message_ix/archive/{zipname}" + path = Path(path) with tempfile.TemporaryDirectory() as td: - print('Retrieving {}'.format(url)) + print(f"Retrieving {url}") zippath = Path(td) / zipname urlretrieve(url, zippath) archive = zipfile.ZipFile(zippath) - print('Unzipping {} to {}'.format(zippath, path)) + print(f"Unzipping {zippath} to {path}") path.mkdir(parents=True, exist_ok=True) - archive.extractall(path) + + # Extract only tutorial files + archive.extractall( + path, + members=filter(lambda n: "/tutorial/" in n, archive.namelist()) + ) # Close *zipfile* so it can be deleted with *td* archive.close() diff --git a/message_ix/tests/test_cli.py b/message_ix/tests/test_cli.py index 28c0baa87..236be34a5 100644 --- a/message_ix/tests/test_cli.py +++ b/message_ix/tests/test_cli.py @@ -29,17 +29,19 @@ def test_copy_model(message_ix_cli, tmp_path, tmp_env): @pytest.mark.parametrize('opts', [ - # During release prep, 'dl' will try to download e.g. v2.0.0, which does - # not yet exist; so the test fails. Use this line: - # pytest.param('', marks=pytest.mark.xfail), - # Otherwise (normal state on master), use this line: '', '--branch=master', - '--tag=1.2.0', + '--tag=v1.2.0', + # Nonexistent tag + pytest.param('--tag=v999', marks=pytest.mark.xfail(raises=AssertionError)), ]) def test_dl(message_ix_cli, opts, tmp_path): r = message_ix_cli('dl', opts, str(tmp_path)) + if r.exit_code != 0: # Debugging information print(r.exception, r.output) - assert r.exit_code == 0 + assert False + + if opts == "": + assert "Default: latest release v2.0.0" in r.output From d54958911a97f0ca66331add56743d30de28b660 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 29 May 2020 17:01:49 +0200 Subject: [PATCH 07/16] Add warnings/disclaimer for MACRO issues i.e. #315, #316, #317, #318, #319, #320 stemming from #223 --- doc/source/api.rst | 15 +++++++++++++++ message_ix/core.py | 1 + 2 files changed, 16 insertions(+) diff --git a/doc/source/api.rst b/doc/source/api.rst index 251c53c62..44836a7a9 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -47,6 +47,7 @@ Support for R usage of the core classes is provided through the `reticulate`_ pa .. autoclass:: message_ix.Scenario :members: + :exclude-members: add_macro :show-inheritance: :inherited-members: @@ -63,6 +64,7 @@ Support for R usage of the core classes is provided through the `reticulate`_ pa add_cat add_horizon + add_macro add_spatial_sets cat cat_list @@ -76,6 +78,11 @@ Support for R usage of the core classes is provided through the `reticulate`_ pa vintage_and_active_years years_active + .. automethod:: add_macro + + .. warning:: MACRO support via :meth:`add_macro` is **experimental** in message_ix 3.0 and may not function as expected on all possible |MESSAGEix| models. + See `a list of known and pending issues `_ on GitHub. + Model classes ------------- @@ -131,7 +138,13 @@ Model classes **var_list** :obj:`None` ================== === + .. autoclass:: MACRO + :members: + :show-inheritance: + + .. autoattribute:: name + .. autoclass:: MESSAGE_MACRO :members: @@ -150,6 +163,8 @@ Model classes - **max_iteration** (:class:`int`, default 50): the maximum number of iterations between the two models. If the solution does not converge after this many iterations, the linked model run fails and no valid result is produced. + .. seealso:: :meth:`.Scenario.add_macro` + .. autoattribute:: name .. autoclass:: GAMSModel diff --git a/message_ix/core.py b/message_ix/core.py index 633b190a7..293e84676 100755 --- a/message_ix/core.py +++ b/message_ix/core.py @@ -448,6 +448,7 @@ def solve(self, model='MESSAGE', solve_options={}, **kwargs): super().solve(model=model, solve_options=solve_options, **kwargs) def add_macro(self, data, scenario=None, check_convergence=True, **kwargs): + """Add MACRO parametrization to the Scenario and calibrate.""" # TODO document from .macro import EXPERIMENTAL, add_model_data, calibrate from .models import MACRO From d4f6603133bbe55e6fad99e6e4a9c7bd71a82785 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 1 Jun 2020 23:48:46 +0200 Subject: [PATCH 08/16] Use non-legacy urlopen() instead of urlretrieve() in "message-ix dl" --- .travis.yml | 5 ++- message_ix/cli.py | 71 +++++++++++++++++++++++------------- message_ix/tests/test_cli.py | 1 + 3 files changed, 50 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index ea56bb184..1f6cb8fa6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,9 @@ language: minimal env: global: - PYENV=py37 - # Encrypted values for MESSAGE_IX_CI_USER, MESSAGE_IX_CI_PW - - secure: "YlqCqO3t4XZbSjP7FseXGU0PWKxCTD+1kBwgRsA829yphaK0UEgDoCHVn0JvE1VDtb4E58eO0Ulnpc0MWhUDGe9dX7o5lpSufKGD+GLl/X00hYpahu17krxphAnqIyATQnY1RzQaqPGThkcx5SVJal1LAqNy5TzsLkYoCq7p1H0G4QlK9WeItlF/FLhrN27Zbi6/d1XYxgEataSHLfm7fb/uQJPj35r/DRwdyhmoDEOmtvbOL4+OQdLfdB56d1YLs5AIK2mMt/DhAoZvrwqWiway+87vxUuY/YIRVtgeukxxEhlp982YC6IFZdz149TS5107fAJpiTEUX2mdjNuzOcBGh6MTMwGVRsBSBfFPsmt9QF5Smwq6KVf9uxm0Sgwz5ih828Tg18YH5iYJ84Y1jF8l7kJw7N1nFKBYoTVB5un9z0iY3Dnw6LQDzJCvNGmdd4Ct38/vTj73Ltcr5hSzcDeE9VQs99z8gM1FCb45MbSYjg00YgfCzypuG5LibTmay89YcTRMIQ9OliQRCJyOB8l19Z1HYMn33nX8BN6IocWKSkeJexV1iNSPt0GZEnBRAccb7iJd/KD/tUeQvVtFbwJ/qSiQOduUb/GMEvcldXQeqsAQRSMvCdadlU2aQGjQF2XQJGlPhQf744RnevrTPm4HeIpvqLTeVvgJiF3h7Pw=" + # Encrypted values for MESSAGE_IX_CI_USER, MESSAGE_IX_CI_PW, + # MESSAGE_IX_GH_USER, MESSAGE_IX_GH_PW + - secure: "IiNG8HMAFIrkeXnVMocSbAtXKgJ7SrZ9VduuAtdftImyBovbs2whtII5rL7qBg47mibREvY0ylUglmc5eLweEbZJO1zOVvNb8urk+cN4aljw9OubAlkpines9R0alhE1M2cOx94JYrNLPJyvMC8s40dVgCJzOM2kyP+cuf8Sr0vCgCAmbtsHSp7f3pkbvR+U7nSrPSHa/LHGtbQ2XJnhd8vvGeAn0oiP2c/Y6yCRhwI+EZYvh0uFSTf4XH6v/vs1zS+2qmWDMe5CUxXgXPtkPQqBhTmQDlbZXcnyoBIWtrpNAP8lXYv6piddizCg+HGpGkmef2pAnFwekQ6tCuQGyyamLlj6nFkljXHn1lbMoFPbRuH4N1vgh8h95r3HUo/Nfxdsza/PEs5HCOA03dU7rE6+ZrV5o0NRGf6cw0lR2evAmPBvkR66soWf3798T5JuUZib3fJ7XlBnhYisKw4Kr+IA6CJTlLVGZ1yd8YcW0bgUt881/kzC5lLjZEN3Is6PL5pkQO9H8tBLKuT0E+X5k6rJlxmuCtQawrpz+0LkoRjXWczH3SY4X7QU1N1Z7SGybNgsJCtwxYinq6QV1vHy2yhJjZtfn3iXtdU7mVyfGtOa3ZVkTO0MDWeDS2FiELz1hMV3zUC5rTZTkV4wLA88Cdijx9fiaW6tirVKhvMSwnI=" os: - linux diff --git a/message_ix/cli.py b/message_ix/cli.py index a464657c0..2f7fd2176 100644 --- a/message_ix/cli.py +++ b/message_ix/cli.py @@ -1,7 +1,9 @@ +from base64 import b64encode import json +import os from pathlib import Path from shutil import copyfile -from urllib.request import urlretrieve +from urllib.request import Request, urlopen import tempfile import zipfile @@ -84,17 +86,39 @@ def copy_model(path, overwrite, set_default): def dl(branch, tag, path): if tag and branch: raise click.BadOptionUsage('Can only provide one of `tag` or `branch`') - - if tag or branch is None: + elif branch: + # Construct URL and filename from branch + zipname = f"{branch}.zip" + url = f"https://github.com/iiasa/message_ix/archive/{zipname}" + else: # Get tag information using GitHub API - url = "https://api.github.com/repos/iiasa/message_ix/tags" - with open(urlretrieve(url)[0]) as f: - tags_info = json.load(f) + args = dict( + url="https://api.github.com/repos/iiasa/message_ix/tags", + headers=dict(), + ) + try: + # Only for Travis/macOS: GitHub rate limits unathenticated API + # requests by IP address. Because the build worker shares an IP, + # the limit is exceeded and the request fails. Use HTTP Basic Auth + # with an encrypted username and password from .travis.yml for a + # higher rate limit. + auth_bytes = b64encode( + "{MESSAGE_IX_GH_USER}:{MESSAGE_IX_GH_PW}" + .format(**os.environ) + .encode() + ) + args["headers"]["Authorization"] = f"Basic {auth_bytes.decode()}" + except KeyError: + pass + + with urlopen(Request(**args)) as response: + tags_info = json.load(response) if tag is None: tag = tags_info[0]["name"] print(f"Default: latest release {tag}") + # Get the zipball URL for the matching tag url = None for info in tags_info: if info["name"] == tag: @@ -105,31 +129,28 @@ def dl(branch, tag, path): raise ValueError(f"tag {repr(tag)} does not exist") zipname = f"{tag}.zip" - else: - # Construct URL and filename from branch - zipname = f"{branch}.zip" - url = f"https://github.com/iiasa/message_ix/archive/{zipname}" - - path = Path(path) + # Context manager to remove the TemporaryDirectory when complete with tempfile.TemporaryDirectory() as td: - print(f"Retrieving {url}") + # Path for zip file zippath = Path(td) / zipname - urlretrieve(url, zippath) - archive = zipfile.ZipFile(zippath) + print(f"Retrieving {url}") + with urlopen(url) as response: # Unauthenticated request + zippath.write_bytes(response.read()) + # Context manager to close the ZipFile when done, so it can be removed print(f"Unzipping {zippath} to {path}") - path.mkdir(parents=True, exist_ok=True) - - # Extract only tutorial files - archive.extractall( - path, - members=filter(lambda n: "/tutorial/" in n, archive.namelist()) - ) - - # Close *zipfile* so it can be deleted with *td* - archive.close() + with zipfile.ZipFile(zippath) as archive: + # Zip archive successfully opened; create the output path + path = Path(path) + path.mkdir(parents=True, exist_ok=True) + + # Extract only tutorial files + archive.extractall( + path, + members=filter(lambda n: "/tutorial/" in n, archive.namelist()) + ) # Add subcommands diff --git a/message_ix/tests/test_cli.py b/message_ix/tests/test_cli.py index 236be34a5..71aa29df1 100644 --- a/message_ix/tests/test_cli.py +++ b/message_ix/tests/test_cli.py @@ -42,6 +42,7 @@ def test_dl(message_ix_cli, opts, tmp_path): # Debugging information print(r.exception, r.output) assert False + print(r.exception, r.output) if opts == "": assert "Default: latest release v2.0.0" in r.output From 9ccf4c6f43aaa29e8fe2421565222dbd75a848da Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 5 Jun 2020 19:28:31 +0200 Subject: [PATCH 09/16] Move MESSAGE_master.gms to desired location; update install docs --- INSTALL.rst | 27 ++++++++++++------- message_ix/model/MESSAGE-MACRO_run.gms | 0 ...master_template.gms => MESSAGE_master.gms} | 0 ...oject_template.gpr => MESSAGE_project.gpr} | 0 4 files changed, 18 insertions(+), 9 deletions(-) mode change 100755 => 100644 message_ix/model/MESSAGE-MACRO_run.gms rename message_ix/model/{templates/MESSAGE_master_template.gms => MESSAGE_master.gms} (100%) rename message_ix/model/{templates/MESSAGE_project_template.gpr => MESSAGE_project.gpr} (100%) diff --git a/INSTALL.rst b/INSTALL.rst index e954f5c48..b0e926777 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -49,12 +49,11 @@ Install |MESSAGEix| from source 3. Install :doc:`ixmp ` from source. -4. (Optional) If you intend to contribute changes to |MESSAGEix|, first register - a Github account, and fork the `message_ix repository `_. This will create a new repository ``/message_ix``. +4. (Optional) If you intend to contribute changes to |MESSAGEix|, first register a Github account, and fork the `message_ix repository `_. + This will create a new repository ``/message_ix``. (Please also see :doc:`contributing`.) -5. Clone either the main repository, or your fork; using the `Github Desktop`_ - client, or the command line:: +5. Clone either the main repository, or your fork; using the `Github Desktop`_ client, or the command line:: $ git clone git@github.com:iiasa/message_ix.git @@ -63,13 +62,23 @@ Install |MESSAGEix| from source 6. Open a command prompt in the ``message_ix`` directory and type:: - $ pip install --editable . + $ pip install --editable .[docs,reporting,tests,tutorial] -7. (Optional) Run the built-in test suite to check that |MESSAGEix| functions - correctly on your system:: + The ``--editable`` flag ensures that changes to the source code are picked up every time ``import message_ix`` is used in Python code. + The ``[docs,reporting,tests,tutorial]`` extra dependencies ensure additional dependencies are installed. + +7. (Optional) If you will be using :file:`MESSAGE_master.gms` outside of Python :mod:`message_ix` to run |MESSAGEix|, you will likely modify this file, but will not want to commit these changes to Git. + Set the Git “assume unchanged” bit for this file:: + + $ git update-index --assume-unchanged message_ix/model/MESSAGE_master.gms + + To unset the bit, use ``--no-assume-unchanged``. + See the `Git documentation `_ for more details. + +8. (Optional) Run the built-in test suite to check that |MESSAGEix| functions correctly on your system:: + + $ pytest - $ pip install --editable .[tests] - $ py.test tests Common issues ------------- diff --git a/message_ix/model/MESSAGE-MACRO_run.gms b/message_ix/model/MESSAGE-MACRO_run.gms old mode 100755 new mode 100644 diff --git a/message_ix/model/templates/MESSAGE_master_template.gms b/message_ix/model/MESSAGE_master.gms similarity index 100% rename from message_ix/model/templates/MESSAGE_master_template.gms rename to message_ix/model/MESSAGE_master.gms diff --git a/message_ix/model/templates/MESSAGE_project_template.gpr b/message_ix/model/MESSAGE_project.gpr similarity index 100% rename from message_ix/model/templates/MESSAGE_project_template.gpr rename to message_ix/model/MESSAGE_project.gpr From dd26e72916bed14471b404285af313b27f6eca13 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 5 Jun 2020 19:43:32 +0200 Subject: [PATCH 10/16] Update pyam-iamc requirement to 0.6 --- ci/pip-requirements.txt | 2 +- setup.cfg | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/pip-requirements.txt b/ci/pip-requirements.txt index 13d02b307..235a6e0eb 100644 --- a/ci/pip-requirements.txt +++ b/ci/pip-requirements.txt @@ -6,4 +6,4 @@ JPype1 >= 0.7.5 # pyam-iamc # Temporary: see https://github.com/IAMconsortium/pyam/issues/259 -git+git://github.com/IAMconsortium/pyam.git#egg=pyam-iamc +git+git://github.com/IAMconsortium/pyam.git@v0.6.0#egg=pyam-iamc diff --git a/setup.cfg b/setup.cfg index 29f637001..1685b750f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,7 +25,9 @@ setup_requires = [options.extras_require] docs = numpydoc; sphinx >= 3.0; sphinx_rtd_theme; sphinxcontrib-bibtex -reporting = pint; pyam-iamc +reporting = + pint + pyam-iamc >= 0.6 tests = asyncssh; pytest >= 5; requests tutorial = jupyter; matplotlib; plotnine From 787502e60485feb81a6b0e7cdd9b1df396aefbe9 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 5 Jun 2020 19:56:42 +0200 Subject: [PATCH 11/16] Update install instructions --- INSTALL.rst | 99 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 35 deletions(-) diff --git a/INSTALL.rst b/INSTALL.rst index b0e926777..c5c61c34f 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -1,94 +1,123 @@ Installation -============ +************ -Install GAMS ------------- +.. contents:: + :local: + +Install system dependencies +=========================== + +GAMS (required) +--------------- |MESSAGEix| requires `GAMS`_. -1. Download the latest version of `GAMS`_ for your operating system; run the - installer. +1. Download GAMS for your operating system; either the `latest version`_ or `version 29`_ (see note below). - .. note:: MESSAGE-MACRO requires GAMS 24.8.1 or later (see - :attr:`.MESSAGE_MACRO.GAMS_min_version`). +2. Run the installer. -2. Add GAMS to the ``PATH`` environment variable. This is **required** in order - for |MESSAGEix| to run the mathematical model core. +3. Ensure that the ``PATH`` environment variable on your system includes the path to the GAMS program: - on Windows, in the GAMS installer… - - Check the box labeled “Use advanced installation mode.” - - Check the box labeled “Add GAMS directory to PATH environment variable” - on the Advanced Options page. + - Check the box labeled “Use advanced installation mode.” + - Check the box labeled “Add GAMS directory to PATH environment variable” on the Advanced Options page. + + - on other platforms (macOS or Linux), add the following line to a file such as :file:`~/.bash_profile` (macOS), :file:`~/.bashrc`, or :file:`~/.profile`:: + + export PATH=$PATH:/path/to/gams-directory-with-gams-binary + +.. note:: + MESSAGE-MACRO and MACRO require GAMS 24.8.1 or later (see :attr:`.MACRO.GAMS_min_version`) + The latest version is recommended. + + GAMS is proprietary software and requires a license to solve optimization problems. + To run both the :mod:`message_ix` and :mod:`ixmp` tutorials and test suites, a “free demonstration” license is required; the free license is suitable for these small models. + Versions of GAMS up to `version 29`_ include such a license with the installer; since version 30, the free demo license is no longer included, but may be requested via the GAMS website. - - on macOS or Linux, add the following line to your ``.bash_profile`` (macOS) - or ``.bashrc`` (Linux):: +.. note:: + If you only have a license for an older version of GAMS, install both the older and the latest versions. + + +Graphviz (optional) +------------------- + +:meth:`.reporting.Reporter.visualize` uses `Graphviz`_, a program for graph visualization. +Installing message_ix causes the python :mod:`graphviz` package to be installed. +If you want to use :meth:`.visualize` or run the test suite, the Graphviz program itself must also be installed; otherwise it is **optional**. + +If you `Install MESSAGEix via Anaconda`_, Graphviz is installed automatically via `its conda-forge package`_. +For other methods of installation, see the `Graphviz download page`_ for downloads and instructions for your system. - export PATH=$PATH:/path/to/gams-directory-with-gams-binary Install |MESSAGEix| via Anaconda --------------------------------- +================================ -After installing GAMS, we recommend that new users install Anaconda, and then -use it to install |MESSAGEix|. Advanced users may choose to install |MESSAGEix| -from source code (next section). +After installing GAMS, we recommend that new users install Anaconda, and then use it to install |MESSAGEix|. +Advanced users may choose to install |MESSAGEix| from source code (next section). -3. Install Python via `Anaconda`_. We recommend the latest version, i.e., - Python 3.6+. +4. Install Python via `Anaconda`_. + We recommend the latest version; currently Python 3.8. -4. Open a command prompt. We recommend Windows users use the “Anaconda Prompt” - to avoid permissions issues when installing and using |MESSAGEix|. This - program is available in the Windows Start menu after installing Anaconda. +5. Open a command prompt. + We recommend Windows users use the “Anaconda Prompt” to avoid permissions issues when installing and using |MESSAGEix|. + This program is available in the Windows Start menu after installing Anaconda. -5. Install the ``message-ix`` package:: +6. Install the ``message-ix`` package:: $ conda install -c conda-forge message-ix + Install |MESSAGEix| from source -------------------------------- +=============================== -3. Install :doc:`ixmp ` from source. +4. Install :doc:`ixmp ` from source. -4. (Optional) If you intend to contribute changes to |MESSAGEix|, first register a Github account, and fork the `message_ix repository `_. +5. (Optional) If you intend to contribute changes to |MESSAGEix|, first register a Github account, and fork the `message_ix repository `_. This will create a new repository ``/message_ix``. (Please also see :doc:`contributing`.) -5. Clone either the main repository, or your fork; using the `Github Desktop`_ client, or the command line:: +6. Clone either the main repository, or your fork; using the `Github Desktop`_ client, or the command line:: $ git clone git@github.com:iiasa/message_ix.git # or: $ git clone git@github.com:USER/message_ix.git -6. Open a command prompt in the ``message_ix`` directory and type:: +7. Open a command prompt in the ``message_ix`` directory and type:: $ pip install --editable .[docs,reporting,tests,tutorial] The ``--editable`` flag ensures that changes to the source code are picked up every time ``import message_ix`` is used in Python code. The ``[docs,reporting,tests,tutorial]`` extra dependencies ensure additional dependencies are installed. -7. (Optional) If you will be using :file:`MESSAGE_master.gms` outside of Python :mod:`message_ix` to run |MESSAGEix|, you will likely modify this file, but will not want to commit these changes to Git. +8. (Optional) If you will be using :file:`MESSAGE_master.gms` outside of Python :mod:`message_ix` to run |MESSAGEix|, you will likely modify this file, but will not want to commit these changes to Git. Set the Git “assume unchanged” bit for this file:: - $ git update-index --assume-unchanged message_ix/model/MESSAGE_master.gms + $ git update-index --assume-unchanged message_ix/model/MESSAGE_master.gms To unset the bit, use ``--no-assume-unchanged``. See the `Git documentation `_ for more details. -8. (Optional) Run the built-in test suite to check that |MESSAGEix| functions correctly on your system:: +9. (Optional) Run the built-in test suite to check that |MESSAGEix| functions correctly on your system:: $ pytest Common issues -------------- +============= No JVM shared library file (jvm.dll) found -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------ If you get an error containing “No JVM shared library file (jvm.dll) found” when creating a :class:`Platform` object (e.g. ``mp = ix.Platform(driver='HSQLDB')``), it is likely that you need to set the ``JAVA_HOME`` environment variable (see for example `these instructions`_). .. _`GAMS`: http://www.gams.com +.. _`latest version`: https://www.gams.com/download/ +.. _`version 29`: https://www.gams.com/29/ +.. _`Graphviz`: https://www.graphviz.org/ +.. _`its conda-forge package`: https://anaconda.org/conda-forge/graphviz +.. _`Graphviz download page`: https://www.graphviz.org/download/ .. _`Anaconda`: https://www.anaconda.com/distribution/#download-section .. _`ixmp`: https://github.com/iiasa/ixmp .. _`Github Desktop`: https://desktop.github.com From 5713cee534e56a3477315919ce7cf7500ac1571e Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Sun, 7 Jun 2020 14:40:18 +0200 Subject: [PATCH 12/16] Update ixmp requirement to 3.0.0 in setup.cfg --- setup.cfg | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 1685b750f..f246022e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,7 @@ zip_safe = True include_package_data = True install_requires = click - ixmp >= 2.0.0 + ixmp >= 3.0.0 numpy pandas PyYAML @@ -24,12 +24,22 @@ setup_requires = setuptools_scm [options.extras_require] -docs = numpydoc; sphinx >= 3.0; sphinx_rtd_theme; sphinxcontrib-bibtex +docs = + numpydoc + sphinx >= 3.0 + sphinx_rtd_theme + sphinxcontrib-bibtex reporting = pint pyam-iamc >= 0.6 -tests = asyncssh; pytest >= 5; requests -tutorial = jupyter; matplotlib; plotnine +tests = + asyncssh + pytest >= 5 + requests +tutorial = + jupyter + matplotlib + plotnine [options.entry_points] console_scripts = From 0cd18d546966b7b6d5d2aad7a9550c14519e5c0d Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Sun, 7 Jun 2020 14:40:44 +0200 Subject: [PATCH 13/16] Remove doc reference to ixmp.Reporter.read_config (removed in 3.0) --- doc/source/reporting.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/reporting.rst b/doc/source/reporting.rst index 8f854f7c5..df81b93b6 100644 --- a/doc/source/reporting.rst +++ b/doc/source/reporting.rst @@ -237,7 +237,6 @@ These automatic features of :class:`~message_ix.reporting.Reporter` are controll full_key get keys - read_config set_filters visualize write From e0bc5db5195cc3e4bc39dbc5bca641534f793aad Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Sun, 7 Jun 2020 14:42:31 +0200 Subject: [PATCH 14/16] Use lualatex for docs PDF build --- doc/source/conf.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/source/conf.py b/doc/source/conf.py index 50b93b1b1..4c9fa4d63 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -90,6 +90,11 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] +# -- Options for LaTeX output ------------------------------------------------- + +# The LaTeX engine to build the docs. +latex_engine = 'lualatex' + # -- Options for sphinx.ext.extlinks ------------------------------------------ @@ -103,6 +108,9 @@ intersphinx_mapping = { 'dask': ('https://docs.dask.org/en/stable/', None), 'ixmp': ('https://message.iiasa.ac.at/projects/ixmp/en/latest/', None), + # For a local build, uncomment and use the following line with a path to + # the directory containing built HTML documentation for ixmp: + # 'ixmp': ('/home/user/path-to-ixmp/doc/build/html', None), 'pandas': ('https://pandas.pydata.org/pandas-docs/stable/', None), 'pint': ('https://pint.readthedocs.io/en/stable/', None), 'pyam': ('https://pyam-iamc.readthedocs.io/en/stable/', None), From 4e852b91be5039c7137bda0b0bee954555dd1788 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Sun, 7 Jun 2020 14:43:15 +0200 Subject: [PATCH 15/16] Update release notes for v3.0.0 --- RELEASE_NOTES.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 7230b80a3..b4710ec8c 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -1,5 +1,7 @@ -Next release -============ +v3.0.0 (2020-06-07) +=================== + +:mod:`message_ix` v3.0.0 coincides with :mod:`ixmp` v3.0.0. Migration notes --------------- @@ -8,6 +10,10 @@ The :ref:`generic storage formulation ` introduces **new ixmp item When loading a Scenario created with a version of `message_ix` older than x.x.x, these items will be initialized (and left empty), using at most one call to :meth:`~message_ix.Scenario.commit`. See :meth:`.MESSAGE.initialize`. +See also the `migration notes for ixmp 3.0.0`_. + +.. _migration notes for ixmp 3.0.0: https://message.iiasa.ac.at/projects/ixmp/en/latest/whatsnew.html#v3-0-0-2020-06-05 + All changes ----------- @@ -27,7 +33,7 @@ All changes v2.0.0 (2020-01-14) =================== -`message_ix` v2.0.0 coincides with `ixmp` v2.0.0. +:mod:`message_ix` v2.0.0 coincides with :mod:`ixmp` v2.0.0. Migration notes --------------- From 3e91a7c25f84368a5eefce9cd38088464f941815 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Sun, 7 Jun 2020 14:55:48 +0200 Subject: [PATCH 16/16] Avoid double builds on AppVeyor & Travis for PR branches --- .appveyor.yml | 10 ++++++++-- .travis.yml | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 6641b6d64..befe651cd 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,5 +1,11 @@ -# Continuous integration procedure for AppVeyor -# Based on https://github.com/rmcgibbo/python-appveyor-conda-example +# Continuous integration configuration for AppVeyor + +branches: + # Do not build PR branches in the main repo; only maintenance branches and + # the merge results of PRs + only: + - master + - 2.0.x environment: matrix: diff --git a/.travis.yml b/.travis.yml index 1f6cb8fa6..bb01d8c1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,13 @@ +# Continuous integration configuration for Travis +# NB use https://config.travis-ci.com/explore to validate changes + +branches: + # Do not build PR branches in the main repo; only maintenance branches and + # the merge results of PRs + only: + - master + - 2.0.x + language: minimal env: