diff --git a/.travis.yml b/.travis.yml index 7988659..324b044 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,13 +12,13 @@ matrix: env: TOX_ENV=py34 - python: 2.7 env: TOX_ENV=py27 - - python: pypy3 - env: TOX_ENV=pypy3 + - python: pypy + env: TOX_ENV=pypy - python: 3.6 env: TOX_ENV=flake8 # Use tox to run tests on Travis-CI to keep one unified method of running tests in any environment -install: +install: - pip install coverage coveralls tox # Command to run tests, e.g. python setup.py test diff --git a/AUTHORS.rst b/AUTHORS.rst index 2f31bac..eaa038a 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -10,5 +10,4 @@ Development Lead Contributors ------------ -* Jake Teo -* Jerome Chan +None yet. Why not be the first? diff --git a/HISTORY.rst b/HISTORY.rst index cb4d47b..b9d17e7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,11 +3,6 @@ History ------- -0.4.11 (2020-03-29) --------------------- - -* Implement '--mode' (Jake Teo, Jerome Chan) - 0.4.8 (2017-06-30) -------------------- diff --git a/README.rst b/README.rst index 07fee33..139fa7d 100644 --- a/README.rst +++ b/README.rst @@ -49,10 +49,7 @@ Usage --force Overwrite existing requirements.txt --diff Compare modules in requirements.txt to project imports. --clean Clean up requirements.txt by removing modules that are not imported in project. - --mode Enables dynamic versioning with , or schemes. - | e.g. Flask~=1.1.2 - | e.g. Flask>=1.1.2 - | e.g. Flask + --no-pin Omit version of output packages. Example ------- @@ -73,5 +70,5 @@ Why not pip freeze? ------------------- - ``pip freeze`` only saves the packages that are installed with ``pip install`` in your environment. -- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project (if you don't have ``virtualenv``). -- and sometimes you just need to create ``requirements.txt`` for a new project without installing modules. +- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project. (if you don't have virtualenv) +- and sometimes you just need to create requirements.txt for a new project without installing modules. diff --git a/pipreqs/mapping b/pipreqs/mapping index c1729eb..6f5a469 100644 --- a/pipreqs/mapping +++ b/pipreqs/mapping @@ -1,4 +1,3 @@ -AFQ:pyAFQ AG_fft_tools:agpy ANSI:pexpect Adafruit:Adafruit_Libraries @@ -14,7 +13,6 @@ Crypto:pycryptodome Cryptodome:pycryptodomex FSM:pexpect FiftyOneDegrees:51degrees_mobile_detector_v3_wrapper -functional:pyfunctional GeoBaseMain:GeoBasesDev GeoBases:GeoBasesDev Globals:Zope2 @@ -24,7 +22,6 @@ Kittens:astro_kittens Levenshtein:python_Levenshtein Lifetime:Zope2 MethodObject:ExtensionClass -MySQLdb:MySQL-python OFS:Zope2 OpenGL:PyOpenGL OpenSSL:pyOpenSSL @@ -595,7 +592,6 @@ devtools:tg.devtools dgis:2gis dhtmlparser:pyDHTMLParser digitalocean:python_digitalocean -discord:discord.py distribute_setup:ez_setup distutils2:Distutils2 django:Django @@ -679,7 +675,6 @@ geventwebsocket:gevent_websocket gflags:python_gflags git:GitPython github:PyGithub -github3:github3.py gitpy:git_py globusonline:globusonline_transfer_api_client google:protobuf @@ -706,7 +701,7 @@ html:pies2overrides htmloutput:nosehtmloutput http:pies2overrides hvad:django_hvad -hydra:hydra-core +krbV:krbv i99fix:199Fix igraph:python_igraph imdb:IMDbPY @@ -732,7 +727,6 @@ keyczar:python_keyczar keyedcache:django_keyedcache keystoneclient:python_keystoneclient kickstarter:kickstart -krbv:krbV kss:kss.core kuyruk:Kuyruk langconv:AdvancedLangConv @@ -804,6 +798,7 @@ msgpack:msgpack_python mutations:aino_mutations mws:amazon_mws mysql:mysql_connector_repackaged +MySQL-python:MySQLdb native_tags:django_native_tags ndg:ndg_httpsclient nereid:trytond_nereid @@ -1004,7 +999,6 @@ ruamel:ruamel.base s2repoze:pysaml2 saga:saga_python saml2:pysaml2 -samtranslator:aws-sam-translator sass:libsass sassc:libsass sasstests:libsass diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py old mode 100644 new mode 100755 index 6c936aa..4b817c3 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -31,11 +31,7 @@ imports. --clean Clean up requirements.txt by removing modules that are not imported in project. - --mode Enables dynamic versioning with , - or schemes. - | e.g. Flask~=1.1.2 - | e.g. Flask>=1.1.2 - | e.g. Flask + --no-pin Omit version of output packages. """ from __future__ import print_function, absolute_import from contextlib import contextmanager @@ -161,25 +157,25 @@ def get_all_imports( return list(packages - data) -def filter_line(line): - return len(line) > 0 and line[0] != "#" +def filter_line(l): + return len(l) > 0 and l[0] != "#" -def generate_requirements_file(path, imports, symbol): +def generate_requirements_file(path, imports): with _open(path, "w") as out_file: logging.debug('Writing {num} requirements: {imports} to {file}'.format( num=len(imports), file=path, imports=", ".join([x['name'] for x in imports]) )) - fmt = '{name}' + symbol + '{version}' + fmt = '{name}=={version}' out_file.write('\n'.join( fmt.format(**item) if item['version'] else '{name}'.format(**item) for item in imports) + '\n') -def output_requirements(imports, symbol): - generate_requirements_file('-', imports, symbol) +def output_requirements(imports): + generate_requirements_file('-', imports) def get_imports_info( @@ -372,11 +368,6 @@ def diff(file_, imports): def clean(file_, imports): """Remove modules that aren't imported in project from file.""" modules_not_imported = compare_modules(file_, imports) - - if len(modules_not_imported) == 0: - logging.info("Nothing to clean in " + file_) - return - re_remove = re.compile("|".join(modules_not_imported)) to_write = [] @@ -401,18 +392,6 @@ def clean(file_, imports): logging.info("Successfully cleaned up requirements in " + file_) -def dynamic_versioning(scheme, imports): - """Enables dynamic versioning with , or schemes.""" - if scheme == "no-pin": - imports = [{"name": item["name"], "version": ""} for item in imports] - symbol = "" - elif scheme == "gt": - symbol = ">=" - elif scheme == "compat": - symbol = "~=" - return imports, symbol - - def init(args): encoding = args.get('--encoding') extra_ignore_dirs = args.get('--ignore') @@ -451,8 +430,6 @@ def init(args): imports = local + get_imports_info(difference, proxy=proxy, pypi_server=pypi_server) - # sort imports based on lowercase name of package, similar to `pip freeze`. - imports = sorted(imports, key=lambda x: x['name'].lower()) path = (args["--savepath"] if args["--savepath"] else os.path.join(input_path, "requirements.txt")) @@ -469,25 +446,18 @@ def init(args): and not args["--savepath"] and not args["--force"] and os.path.exists(path)): - logging.warning("requirements.txt already exists, " + logging.warning("Requirements.txt already exists, " "use --force to overwrite it") return - if args["--mode"]: - scheme = args.get("--mode") - if scheme in ["compat", "gt", "no-pin"]: - imports, symbol = dynamic_versioning(scheme, imports) - else: - raise ValueError("Invalid argument for mode flag, " - "use 'compat', 'gt' or 'no-pin' instead") - else: - symbol = "==" + if args.get('--no-pin'): + imports = [{'name': item["name"], 'version': ''} for item in imports] if args["--print"]: - output_requirements(imports, symbol) + output_requirements(imports) logging.info("Successfully output requirements") else: - generate_requirements_file(path, imports, symbol) + generate_requirements_file(path, imports) logging.info("Successfully saved requirements file in " + path) diff --git a/pipreqs/stdlib b/pipreqs/stdlib index c2c1fdd..470fd5c 100644 --- a/pipreqs/stdlib +++ b/pipreqs/stdlib @@ -133,7 +133,6 @@ curses curses.ascii curses.panel curses.textpad -dataclasses datetime dbhash dbm @@ -371,7 +370,6 @@ robotparser runpy sched ScrolledText -secrets select selectors sets diff --git a/tests/_data/test.py b/tests/_data/test.py index fdb6ec3..cfd039c 100644 --- a/tests/_data/test.py +++ b/tests/_data/test.py @@ -31,10 +31,6 @@ # Nose from nose.importer import Importer, add_path, remove_path # loader.py -# see issue #88 -import analytics -import flask_seasurf - import atexit from __future__ import print_function from docopt import docopt diff --git a/tests/_data_clean/test.py b/tests/_data_clean/test.py deleted file mode 100644 index 8cffb51..0000000 --- a/tests/_data_clean/test.py +++ /dev/null @@ -1,65 +0,0 @@ -"""unused import""" -# pylint: disable=undefined-all-variable, import-error, no-absolute-import, too-few-public-methods, missing-docstring -import xml.etree # [unused-import] -import xml.sax # [unused-import] -import os.path as test # [unused-import] -from sys import argv as test2 # [unused-import] -from sys import flags # [unused-import] -# +1:[unused-import,unused-import] -from collections import deque, OrderedDict, Counter -# All imports above should be ignored -import requests # [unused-import] - -# setuptools -import zipimport # command/easy_install.py - -# twisted -from importlib import invalidate_caches # python/test/test_deprecate.py - -# astroid -import zipimport # manager.py -# IPython -from importlib.machinery import all_suffixes # core/completerlib.py -import importlib # html/notebookapp.py - -from IPython.utils.importstring import import_item # Many files - -# pyflakes -# test/test_doctests.py -from pyflakes.test.test_imports import Test as TestImports - -# Nose -from nose.importer import Importer, add_path, remove_path # loader.py - -# see issue #88 -import analytics -import flask_seasurf - -import atexit -from __future__ import print_function -from docopt import docopt -import curses, logging, sqlite3 -import logging -import os -import sqlite3 -import time -import sys -import signal -import bs4 -import nonexistendmodule -import boto as b, peewee as p -# import django -import flask.ext.somext # # # -# from sqlalchemy import model -try: - import ujson as json -except ImportError: - import json - -import models - - -def main(): - pass - -import after_method_is_valid_even_if_not_pep8 diff --git a/tests/test_pipreqs.py b/tests/test_pipreqs.py old mode 100644 new mode 100755 index f82d3db..dcd75c5 --- a/tests/test_pipreqs.py +++ b/tests/test_pipreqs.py @@ -18,40 +18,22 @@ class TestPipreqs(unittest.TestCase): def setUp(self): - self.modules = [ - 'flask', 'requests', 'sqlalchemy', 'docopt', 'boto', 'ipython', - 'pyflakes', 'nose', 'analytics', 'flask_seasurf', 'peewee', - 'ujson', 'nonexistendmodule', 'bs4', - 'after_method_is_valid_even_if_not_pep8' - ] + self.modules = ['flask', 'requests', 'sqlalchemy', + 'docopt', 'boto', 'ipython', 'pyflakes', 'nose', + 'peewee', 'ujson', 'nonexistendmodule', 'bs4', 'after_method_is_valid_even_if_not_pep8' ] self.modules2 = ['beautifulsoup4'] self.local = ["docopt", "requests", "nose", 'pyflakes'] self.project = os.path.join(os.path.dirname(__file__), "_data") - self.project_clean = os.path.join( - os.path.dirname(__file__), - "_data_clean" - ) - self.project_invalid = os.path.join( - os.path.dirname(__file__), - "_invalid_data" - ) - self.project_with_ignore_directory = os.path.join( - os.path.dirname(__file__), - "_data_ignore" - ) - self.project_with_duplicated_deps = os.path.join( - os.path.dirname(__file__), - "_data_duplicated_deps" - ) + self.project_invalid = os.path.join(os.path.dirname(__file__), "_invalid_data") + self.project_with_ignore_directory = os.path.join(os.path.dirname(__file__), "_data_ignore") + self.project_with_duplicated_deps = os.path.join(os.path.dirname(__file__), "_data_duplicated_deps") self.requirements_path = os.path.join(self.project, "requirements.txt") self.alt_requirement_path = os.path.join( - self.project, - "requirements2.txt" - ) + self.project, "requirements2.txt") def test_get_all_imports(self): imports = pipreqs.get_all_imports(self.project) - self.assertEqual(len(imports), 15) + self.assertEqual(len(imports), 13) for item in imports: self.assertTrue( item.lower() in self.modules, "Import is missing: " + item) @@ -72,8 +54,7 @@ def test_invalid_python(self): """ Test that invalid python files cannot be imported. """ - self.assertRaises( - SyntaxError, pipreqs.get_all_imports, self.project_invalid) + self.assertRaises(SyntaxError, pipreqs.get_all_imports, self.project_invalid) def test_get_imports_info(self): """ @@ -81,9 +62,8 @@ def test_get_imports_info(self): """ imports = pipreqs.get_all_imports(self.project) with_info = pipreqs.get_imports_info(imports) - # Should contain 10 items without the "nonexistendmodule" and - # "after_method_is_valid_even_if_not_pep8" - self.assertEqual(len(with_info), 13) + # Should contain 10 items without the "nonexistendmodule" and "after_method_is_valid_even_if_not_pep8" + self.assertEqual(len(with_info), 11) for item in with_info: self.assertTrue( item['name'].lower() in self.modules, @@ -97,12 +77,10 @@ def test_get_pkg_names(self): def test_get_use_local_only(self): """ - Test without checking PyPI, check to see if names of local - imports matches what we expect + Test without checking PyPI, check to see if names of local imports matches what we expect - Note even though pyflakes isn't in requirements.txt, - It's added to locals since it is a development dependency - for testing + It's added to locals since it is a development dependency for testing """ # should find only docopt and requests imports_with_info = pipreqs.get_import_local(self.modules) @@ -111,28 +89,24 @@ def test_get_use_local_only(self): def test_init(self): """ - Test that all modules we will test upon are in requirements file + Test that all modules we will test upon, are in requirements file """ pipreqs.init({'': self.project, '--savepath': None, '--print': False, '--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None, - '--diff': None, '--clean': None, '--mode': None}) + '--diff': None, '--clean': None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() for item in self.modules[:-3]: self.assertTrue(item.lower() in data) - # It should be sorted based on names. - data = data.strip().split('\n') - self.assertEqual(data, sorted(data)) def test_init_local_only(self): """ - Test that items listed in requirements.text are the same - as locals expected + Test that items listed in requirements.text are the same as locals expected """ pipreqs.init({'': self.project, '--savepath': None, '--print': False, '--use-local': True, '--force': True, '--proxy':None, '--pypi-server':None, - '--diff': None, '--clean': None, '--mode': None}) + '--diff': None, '--clean': None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.readlines() @@ -142,12 +116,11 @@ def test_init_local_only(self): def test_init_savepath(self): """ - Test that we can save requirements.txt correctly - to a different path + Test that we can save requiremnts.tt correctly to a different path """ - pipreqs.init({'': self.project, '--savepath': self.alt_requirement_path, - '--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False, - '--diff': None, '--clean': None, '--mode': None}) + pipreqs.init({'': self.project, '--savepath': + self.alt_requirement_path, '--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False, + "--diff": None, "--clean": None}) assert os.path.exists(self.alt_requirement_path) == 1 with open(self.alt_requirement_path, "r") as f: data = f.read().lower() @@ -158,14 +131,13 @@ def test_init_savepath(self): def test_init_overwrite(self): """ - Test that if requiremnts.txt exists, it will not be - automatically overwritten + Test that if requiremnts.txt exists, it will not automatically be overwritten """ with open(self.requirements_path, "w") as f: f.write("should_not_be_overwritten") - pipreqs.init({'': self.project, '--savepath': None, '--use-local': None, - '--force': None, '--proxy':None, '--pypi-server':None, '--print': False, - '--diff': None, '--clean': None, '--mode': None}) + pipreqs.init({'': self.project, '--savepath': None, + '--use-local': None, '--force': None, '--proxy':None, '--pypi-server':None, '--print': False, + "--diff": None, "--clean": None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() @@ -173,43 +145,35 @@ def test_init_overwrite(self): def test_get_import_name_without_alias(self): """ - Test that function get_name_without_alias() - will work on a string. - - Note: This isn't truly needed when pipreqs is walking - the AST to find imports + Test that function get_name_without_alias() will work on a string. + - Note: This isn't truly needed when pipreqs is walking the AST to find imports """ import_name_with_alias = "requests as R" expected_import_name_without_alias = "requests" import_name_without_aliases = pipreqs.get_name_without_alias( import_name_with_alias) self.assertEqual( - import_name_without_aliases, - expected_import_name_without_alias - ) + import_name_without_aliases, expected_import_name_without_alias) def test_custom_pypi_server(self): """ Test that trying to get a custom pypi sever fails correctly """ - self.assertRaises( - requests.exceptions.MissingSchema, pipreqs.init, - {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy': None, - '--pypi-server': 'nonexistent'} - ) + self.assertRaises(requests.exceptions.MissingSchema, pipreqs.init, {'': self.project, '--savepath': None, '--print': False, + '--use-local': None, '--force': True, '--proxy': None, '--pypi-server': 'nonexistent'}) def test_ignored_directory(self): """ Test --ignore parameter """ pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, - '--print': False, '--use-local': None, '--force': True, - '--proxy':None, '--pypi-server':None, + {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, + '--use-local': None, '--force': True, + '--proxy':None, + '--pypi-server':None, '--ignore':'.ignored_dir,.ignore_second', '--diff': None, - '--clean': None, - '--mode': None + '--clean': None } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: @@ -217,27 +181,9 @@ def test_ignored_directory(self): for item in ['click', 'getpass']: self.assertFalse(item.lower() in data) - def test_dynamic_version_no_pin_scheme(self): - """ - Test --mode=no-pin - """ - pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, - '--print': False, '--use-local': None, '--force': True, - '--proxy': None, '--pypi-server': None, - '--diff': None, - '--clean': None, - '--mode': 'no-pin' - } - ) - with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: - data = f.read().lower() - for item in ['beautifulsoup4', 'boto']: - self.assertTrue(item.lower() in data) - - def test_dynamic_version_gt_scheme(self): + def test_omit_version(self): """ - Test --mode=gt + Test --no-pin parameter """ pipreqs.init( {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, @@ -246,81 +192,13 @@ def test_dynamic_version_gt_scheme(self): '--pypi-server': None, '--diff': None, '--clean': None, - '--mode': 'gt' + '--no-pin': True } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: - data = f.readlines() - for item in data: - symbol = '>=' - message = 'symbol is not in item' - self.assertIn(symbol, item, message) - - def test_dynamic_version_compat_scheme(self): - """ - Test --mode=compat - """ - pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, - '--proxy': None, - '--pypi-server': None, - '--diff': None, - '--clean': None, - '--mode': 'compat' - } - ) - with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: - data = f.readlines() - for item in data: - symbol = '~=' - message = 'symbol is not in item' - self.assertIn(symbol, item, message) - - def test_clean(self): - """ - Test --clean parameter - """ - pipreqs.init( - {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy': None, - '--pypi-server': None, '--diff': None, '--clean': None, - '--mode': None} - ) - assert os.path.exists(self.requirements_path) == 1 - pipreqs.init( - {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': None, '--proxy': None, - '--pypi-server': None, '--diff': None, - '--clean': self.requirements_path, '--mode': 'non-pin'} - ) - with open(self.requirements_path, "r") as f: data = f.read().lower() - for item in self.modules[:-3]: - self.assertTrue(item.lower() in data) - - def test_clean_with_imports_to_clean(self): - """ - Test --clean parameter when there are imports to clean - """ - cleaned_module = 'sqlalchemy' - pipreqs.init( - {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy': None, - '--pypi-server': None, '--diff': None, '--clean': None, - '--mode': None} - ) - assert os.path.exists(self.requirements_path) == 1 - modules_clean = [m for m in self.modules if m != cleaned_module] - pipreqs.init( - {'': self.project_clean, '--savepath': None, - '--print': False, '--use-local': None, '--force': None, - '--proxy': None, '--pypi-server': None, '--diff': None, - '--clean': self.requirements_path, '--mode': 'non-pin'} - ) - with open(self.requirements_path, "r") as f: - data = f.read().lower() - self.assertTrue(cleaned_module not in data) + for item in ['beautifulsoup4==4.8.1', 'boto==2.49.0']: + self.assertFalse(item.lower() in data) def tearDown(self): """ diff --git a/tox.ini b/tox.ini index 554b3c1..28a1dfa 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py34, py35, py36, pypy3, flake8 +envlist = py27, py34, py35, py36, pypy, flake8 [testenv] setenv =