Install Datumaro as package (#2163)

* Remove Datumaro sources

* Install Datumaro as package

* Update Datumaro links

* fix

* remove unnecessary dependencies

* Update travis

* update coverage config

.coveragerc

@@ -3,13 +3,10 @@ branch = true
 # relative_files = true # does not work?
 
 source =
-    datumaro/datumaro/
     cvat/apps/
     utils/cli/
 
 omit =
-    datumaro/datumaro/__main__.py
-    datumaro/datumaro/version.py
     cvat/settings/*
     */tests/*
     */test_*

.travis.yml

@@ -1,5 +1,3 @@
-sudo: required
-
 language: python
 
 python:
@@ -36,7 +34,7 @@ before_script:
 
 script:
   # FIXME: Git package and application name conflict in PATH and try to leave only one python test execution
-  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps && coverage run -a manage.py test --pattern="_test*.py" cvat/apps/dataset_manager/tests cvat/apps/engine/tests utils/cli && coverage run -a manage.py test datumaro/ && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'
+  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps && coverage run -a manage.py test --pattern="_test*.py" cvat/apps/dataset_manager/tests cvat/apps/engine/tests utils/cli && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'
   - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-data && npm install && cd ../cvat-core && npm install && npm run test && coveralls-lcov -v -n ./reports/coverage/lcov.info > ${CONTAINER_COVERAGE_DATA_DIR}/coverage.json'
   # Up all containers
   - docker-compose up -d

.vscode/settings.json

@@ -25,13 +25,7 @@
         }
     ],
     "python.linting.pylintEnabled": true,
-    "python.envFile": "${workspaceFolder}/.vscode/python.env",
     "python.testing.unittestEnabled": true,
-    "python.testing.unittestArgs": [
-        "-v",
-        "-s",
-        "./datumaro",
-    ],
     "licenser.license": "Custom",
     "licenser.customHeader": "Copyright (C) @YEAR@ Intel Corporation\n\nSPDX-License-Identifier: MIT",
     "files.trimTrailingWhitespace": true

Dockerfile

@@ -73,8 +73,6 @@ COPY components /tmp/components
 COPY cvat/requirements/ /tmp/requirements/
 COPY supervisord.conf mod_wsgi.conf wait-for-it.sh manage.py ${HOME}/
 RUN python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt
-# pycocotools package is impossible to install with its dependencies by one pip install command
-RUN python3 -m pip install --no-cache-dir pycocotools==2.0.0
 
 ARG CLAM_AV
 ENV CLAM_AV=${CLAM_AV}
@@ -95,9 +93,6 @@ COPY cvat/ ${HOME}/cvat
 COPY cvat-core/ ${HOME}/cvat-core
 COPY cvat-data/ ${HOME}/cvat-data
 COPY tests ${HOME}/tests
-COPY datumaro/ ${HOME}/datumaro
-
-RUN python3 -m pip install --no-cache-dir -r ${HOME}/datumaro/requirements.txt
 
 RUN chown -R ${USER}:${USER} .
 

Dockerfile.ci

@@ -9,7 +9,7 @@ RUN apt-get update && \
         apt-utils \
         build-essential \
         python3-dev \
-        ruby \ 
+        ruby \
         && \
     curl https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
     echo 'deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main' | tee /etc/apt/sources.list.d/google-chrome.list && \

README.md

@@ -41,15 +41,16 @@ annotation team. Try it online [cvat.org](https://cvat.org).
 ## Supported annotation formats
 
 Format selection is possible after clicking on the Upload annotation
-and Dump annotation buttons. [Datumaro](datumaro/README.md) dataset
+and Dump annotation buttons.
+[Datumaro](https://github.com/openvinotoolkit/datumaro/README.md) dataset
 framework allows additional dataset transformations
 via its command line tool and Python library.
 
 | Annotation format                                                                          | Import | Export |
 | ------------------------------------------------------------------------------------------ | ------ | ------ |
 | [CVAT for images](cvat/apps/documentation/xml_format.md#annotation)                        | X      | X      |
 | [CVAT for a video](cvat/apps/documentation/xml_format.md#interpolation)                    | X      | X      |
-| [Datumaro](datumaro/README.md)                                                             |        | X      |
+| [Datumaro](https://github.com/openvinotoolkit/datumaro)                                    |        | X      |
 | [PASCAL VOC](http://host.robots.ox.ac.uk/pascal/VOC/)                                      | X      | X      |
 | Segmentation masks from [PASCAL VOC](http://host.robots.ox.ac.uk/pascal/VOC/)              | X      | X      |
 | [YOLO](https://pjreddie.com/darknet/yolo/)                                                 | X      | X      |

cvat/apps/dataset_manager/formats/datumaro/__init__.py

@@ -11,7 +11,7 @@
 from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor,
     import_dm_annotations)
 from cvat.apps.dataset_manager.util import make_zip_archive
-from cvat.settings.base import BASE_DIR, DATUMARO_PATH
+from cvat.settings.base import BASE_DIR
 from datumaro.components.project import Project
 
 from ..registry import dm_env, exporter
@@ -79,14 +79,7 @@ def _export(self, task_data, save_dir, save_images=False):
                 osp.join(templates_dir, self._REMOTE_IMAGES_EXTRACTOR + '.py'),
                 osp.join(target_dir, self._REMOTE_IMAGES_EXTRACTOR + '.py'))
 
-        # Make Datumaro and CVAT CLI modules available to the user
-        shutil.copytree(DATUMARO_PATH, osp.join(save_dir, 'datumaro'),
-            ignore=lambda src, names: ['__pycache__'] + [
-                n for n in names
-                if sum([int(n.endswith(ext)) for ext in
-                        ['.pyx', '.pyo', '.pyd', '.pyc']])
-            ])
-
+        # Make CVAT CLI module available to the user
         cvat_utils_dst_dir = osp.join(save_dir, 'cvat', 'utils')
         os.makedirs(cvat_utils_dst_dir)
         shutil.copytree(osp.join(BASE_DIR, 'utils', 'cli'),

cvat/apps/dataset_manager/formats/datumaro/export_templates/README.md

@@ -6,7 +6,7 @@ python -m virtualenv .venv
 . .venv/bin/activate
 
 # install dependencies
-pip install -e datumaro/
+pip install 'git+https://github.com/openvinotoolkit/datumaro'
 pip install -r cvat/utils/cli/requirements.txt
 
 # set up environment
@@ -17,4 +17,4 @@ export PYTHONPATH
 datum --help
 ```
 
-Check Datumaro [docs](datumaro/README.md) for more info.
+Check [Datumaro docs](https://github.com/openvinotoolkit/datumaro/README.md) for more info.

cvat/requirements/base.txt

@@ -44,4 +44,5 @@ tensorflow==2.2.0 # Optional requirement of Datumaro
 # The package is used by pyunpack as a command line tool to support multiple
 # archives. Don't use as a python module because it has GPL license.
 patool==1.12
-diskcache==5.0.2
+diskcache==5.0.2
+git+https://github.com/openvinotoolkit/datumaro@v0.1.0

cvat/settings/base.py

@@ -409,9 +409,6 @@ def generate_ssh_keys():
 LOCAL_LOAD_MAX_FILES_COUNT = 500
 LOCAL_LOAD_MAX_FILES_SIZE = 512 * 1024 * 1024  # 512 MB
 
-DATUMARO_PATH = os.path.join(BASE_DIR, 'datumaro')
-sys.path.append(DATUMARO_PATH)
-
 RESTRICTIONS = {
     'user_agreements': [],