Improve documentation around index restrictions (#5029)

* Improve documentation around index restrictions

* Update docs/advanced.rst

* Refine index documentation updates.   Factor out and re-use method before closing down other PR.

* Fully remove the --extra-index-url argument

Co-authored-by: Yusuke Nishioka <yusuke.nishioka.0713@gmail.com>

docs/advanced.rst

@@ -17,31 +17,54 @@ This document covers some of Pipenv's more glorious and advanced features.
 ☤ Specifying Package Indexes
 ----------------------------
 
-If you'd like a specific package to be installed with a specific package index, you can do the following::
+Starting in release ``2022.3.23`` all packages are mapped only to a single package index for security reasons.
+All unspecified packages are resolved using the default index source; the default package index is PyPI.
+
+For a specific package to be installed from an alternate package index, you must match the name of the index as in the following example::
 
     [[source]]
     url = "https://pypi.org/simple"
     verify_ssl = true
     name = "pypi"
 
     [[source]]
-    url = "http://pypi.home.kennethreitz.org/simple"
+    url = "https://download.pytorch.org/whl/cu113/"
     verify_ssl = false
-    name = "home"
+    name = "pytorch"
 
     [dev-packages]
 
     [packages]
-    requests = {version="*", index="home"}
-    maya = {version="*", index="pypi"}
-    records = "*"
+    torch = {version="*", index="pytorch"}
+    numpy = {version="*"}
+
+You may install a package such as the example ``torch`` from the named index ``pytorch`` using the CLI by running
+the following command:
+
+``pipenv install --index=pytorch torch``
+
+Alternatively the index may be specified by full url, and it will be added to the ``Pipfile`` with a generated name
+unless it already exists in which case the existing name with be reused when pinning the package index.
+
+**Note:** In prior versions of ``pipenv`` you could specify ``--extra-index-urls`` to the ``pip`` resolver and avoid
+specifically matching the expected index by name.   That functionality was deprecated in favor of index restricted
+packages, which is a simplifying assumption that is more security mindful.  The pip documentation has the following
+warning around the ``--extra-index-urls`` option::
+
+> Using this option to search for packages which are not in the main repository (such as private packages) is unsafe,
+per a security vulnerability called dependency confusion: an attacker can claim the package on the public repository
+in a way that will ensure it gets chosen over the private package.
 
-Very fancy.
+Should you wish to use an alternative default index other than PyPI: simply do not specify PyPI as one of the
+sources in your ``Pipfile``.  When PyPI is omitted, then any public packages required either directly or
+as sub-dependencies must be mirrored onto your private index or they will not resolve properly.  This matches the
+standard recommendation of ``pip`` maintainers: "To correctly make a private project installable is to point
+--index-url to an index that contains both PyPI and their private projects—which is our recommended best practice."
 
 ☤ Using a PyPI Mirror
 ----------------------------
 
-If you would like to override the default PyPI index URLs with the URL for a PyPI mirror, you can use the following::
+Should you wish to override the default PyPI index URLs with the URL for a PyPI mirror, you can do the following::
 
     $ pipenv install --pypi-mirror <mirror_url>
 
@@ -53,9 +76,9 @@ If you would like to override the default PyPI index URLs with the URL for a PyP
 
     $ pipenv uninstall --pypi-mirror <mirror_url>
 
-Alternatively, you can set the ``PIPENV_PYPI_MIRROR`` environment variable.
+Alternatively, setting the ``PIPENV_PYPI_MIRROR`` environment variable is equivalent to passing ``--pypi-mirror <mirror_url>``.
 
-☤ Injecting credentials into Pipfiles via environment variables
+☤ Injecting credentials into Pipfile via environment variables
 -----------------------------------------------------------------
 
 Pipenv will expand environment variables (if defined) in your Pipfile. Quite

news/5022.doc.rst

@@ -0,0 +1 @@
+Improve documentation around extra indexes and index restricted packages.

news/5022.removal.rst

@@ -0,0 +1,2 @@
+Removes the optional ``install`` argument ``--extra-index-url`` as it was not compatible with index restricted packages.
+Using the ``--index`` argument is the correct way to specify a package should be pulled from the non-default index.

news/5022.trivial.rst

@@ -0,0 +1 @@
+Reuse existing utility method for determining if index is pypi, reducing code complexity.

pipenv/cli/command.py

@@ -236,7 +236,6 @@ def install(state, **kwargs):
         keep_outdated=state.installstate.keep_outdated,
         selective_upgrade=state.installstate.selective_upgrade,
         index_url=state.index,
-        extra_index_url=state.extra_index_urls,
         packages=state.installstate.packages,
         editable_packages=state.installstate.editables,
         site_packages=state.site_packages,

pipenv/cli/options.py

@@ -59,7 +59,6 @@ def main(self, *args, **kwargs):
 class State:
     def __init__(self):
         self.index = None
-        self.extra_index_urls = []
         self.verbose = False
         self.quiet = False
         self.pypi_mirror = None
@@ -111,28 +110,12 @@ def callback(ctx, param, value):
         "--index",
         expose_value=False,
         envvar="PIP_INDEX_URL",
-        help="Target PyPI-compatible package index url.",
+        help="Specify target package index by url or index name from Pipfile.",
         nargs=1,
         callback=callback,
     )(f)
 
 
-def extra_index_option(f):
-    def callback(ctx, param, value):
-        state = ctx.ensure_object(State)
-        state.extra_index_urls.extend(list(value))
-        return value
-
-    return option(
-        "--extra-index-url",
-        multiple=True,
-        expose_value=False,
-        help="URLs to the extra PyPI compatible indexes to query for package look-ups.",
-        callback=callback,
-        envvar="PIP_EXTRA_INDEX_URL",
-    )(f)
-
-
 def editable_option(f):
     def callback(ctx, param, value):
         state = ctx.ensure_object(State)
@@ -630,7 +613,6 @@ def sync_options(f):
 def install_options(f):
     f = sync_options(f)
     f = index_option(f)
-    f = extra_index_option(f)
     f = requirementstxt_option(f)
     f = selective_upgrade_option(f)
     f = ignore_pipfile_option(f)

pipenv/core.py

@@ -1923,7 +1923,6 @@ def do_install(
     packages=False,
     editable_packages=False,
     index_url=False,
-    extra_index_url=False,
     dev=False,
     three=False,
     python=False,
@@ -2165,7 +2164,6 @@ def do_install(
                         pre=pre,
                         requirements_dir=requirements_directory,
                         index=index_url,
-                        extra_indexes=extra_index_url,
                         pypi_mirror=pypi_mirror,
                     )
                     if c.returncode:
@@ -2240,13 +2238,11 @@ def do_install(
                     )
                 )
                 # Add the package to the Pipfile.
-                indexes = list(filter(None, [index_url, *extra_index_url]))
-                for index in indexes:
+                if index_url:
                     index_name = project.add_index_to_pipfile(
-                        index, verify_ssl=index.startswith("https:")
+                        index_url, verify_ssl=index_url.startswith("https:")
                     )
-                    if index_url and not extra_index_url:
-                        pkg_requirement.index = index_name
+                    pkg_requirement.index = index_name
                 try:
                     project.add_package_to_pipfile(pkg_requirement, dev)
                 except ValueError:

pipenv/patched/notpip/_internal/cli/req_command.py

@@ -62,9 +62,6 @@ def _get_index_urls(cls, options: Values) -> Optional[List[str]]:
             url = getattr(options, "index_url", None)
             if url:
                 index_urls.append(url)
-        urls = getattr(options, "extra_index_urls", None)
-        if urls:
-            index_urls.extend(urls)
         # Return None rather than an empty list
         return index_urls or None
 

pipenv/patched/notpip/_internal/req/req_file.py

@@ -225,8 +225,6 @@ def handle_option_line(
             index_urls = [opts.index_url]
         if opts.no_index is True:
             index_urls = []
-        if opts.extra_index_urls:
-            index_urls.extend(opts.extra_index_urls)
         if opts.find_links:
             # FIXME: it would be nice to keep track of the source
             # of the find_links: support a find-links local path

pipenv/utils/resolver.py

@@ -26,7 +26,7 @@
     translate_markers,
 )
 from .indexes import parse_indexes, prepare_pip_source_args
-from .internet import _get_requests_session
+from .internet import _get_requests_session, is_pypi_url
 from .locking import format_requirement_for_lockfile, prepare_lockfile
 from .shell import make_posix, subprocess_run, temp_environ
 from .spinner import create_spinner
@@ -744,10 +744,7 @@ def collect_hashes(self, ireq):
             sources = list(
                 filter(lambda s: s.get("name") == self.index_lookup[ireq.name], sources)
             )
-        if any(
-            "python.org" in source["url"] or "pypi.org" in source["url"]
-            for source in sources
-        ):
+        if any(is_pypi_url(source["url"]) for source in sources):
             hashes = self._get_hashes_from_pypi(ireq)
             if hashes:
                 return hashes