assets.py

"""
Asset compilation and collection.
"""


import argparse
import glob
import json
import os
import traceback
from datetime import datetime
from functools import wraps
from threading import Timer

from paver import tasks
from paver.easy import call_task, cmdopts, consume_args, needs, no_help, path, sh, task
from watchdog.events import PatternMatchingEventHandler
from watchdog.observers import Observer
from watchdog.observers.api import DEFAULT_OBSERVER_TIMEOUT

from openedx.core.djangoapps.theming.paver_helpers import get_theme_paths

from .utils.cmd import cmd, django_cmd
from .utils.envs import Env
from .utils.process import run_background_process
from .utils.timer import timed

# setup baseline paths

ALL_SYSTEMS = ['lms', 'studio']

LMS = 'lms'
CMS = 'cms'

SYSTEMS = {
    'lms': LMS,
    'cms': CMS,
    'studio': CMS
}

# Common lookup paths that are added to the lookup paths for all sass compilations
COMMON_LOOKUP_PATHS = [
    path("common/static"),
    path("common/static/sass"),
    path('node_modules/@edx'),
    path('node_modules'),
]

# A list of NPM installed libraries that should be copied into the common
# static directory.
# If string ends with '/' then all file in the directory will be copied.
NPM_INSTALLED_LIBRARIES = [
    'backbone.paginator/lib/backbone.paginator.js',
    'backbone/backbone.js',
    'bootstrap/dist/js/bootstrap.bundle.js',
    'hls.js/dist/hls.js',
    'jquery-migrate/dist/jquery-migrate.js',
    'jquery.scrollto/jquery.scrollTo.js',
    'jquery/dist/jquery.js',
    'moment-timezone/builds/moment-timezone-with-data.js',
    'moment/min/moment-with-locales.js',
    'picturefill/dist/picturefill.js',
    'requirejs/require.js',
    'underscore.string/dist/underscore.string.js',
    'underscore/underscore.js',
    '@edx/studio-frontend/dist/',
    'which-country/index.js'
]

# A list of NPM installed developer libraries that should be copied into the common
# static directory only in development mode.
NPM_INSTALLED_DEVELOPER_LIBRARIES = [
    'sinon/pkg/sinon.js',
    'squirejs/src/Squire.js',
]

# Directory to install static vendor files
NPM_JS_VENDOR_DIRECTORY = path('common/static/common/js/vendor')
NPM_CSS_VENDOR_DIRECTORY = path("common/static/common/css/vendor")
NPM_CSS_DIRECTORY = path("common/static/common/css")

# system specific lookup path additions, add sass dirs if one system depends on the sass files for other systems
SASS_LOOKUP_DEPENDENCIES = {
    'cms': [path('lms') / 'static' / 'sass' / 'partials', ],
}

# Collectstatic log directory setting
COLLECTSTATIC_LOG_DIR_ARG = 'collect_log_dir'

# Webpack command
WEBPACK_COMMAND = 'STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} $(npm bin)/webpack {options}'


def get_sass_directories(system, theme_dir=None):
    """
    Determine the set of SASS directories to be compiled for the specified list of system and theme
    and return a list of those directories.

    Each item in the list is dict object containing the following key-value pairs.
    {
        "sass_source_dir": "",  # directory where source sass files are present
        "css_destination_dir": "",  # destination where css files would be placed
        "lookup_paths": [],  # list of directories to be passed as lookup paths for @import resolution.
    }

    if theme_dir is empty or None then return sass directories for the given system only. (i.e. lms or cms)

    :param system: name if the system for which to compile sass e.g. 'lms', 'cms'
    :param theme_dir: absolute path of theme for which to compile sass files.
    """
    if system not in SYSTEMS:
        raise ValueError("'system' must be one of ({allowed_values})".format(
            allowed_values=', '.join(list(SYSTEMS.keys())))
        )
    system = SYSTEMS[system]

    applicable_directories = []

    if theme_dir:
        # Add theme sass directories
        applicable_directories.extend(
            get_theme_sass_dirs(system, theme_dir)
        )
    else:
        # add system sass directories
        applicable_directories.extend(
            get_system_sass_dirs(system)
        )

    return applicable_directories


def get_common_sass_directories():
    """
    Determine the set of common SASS directories to be compiled for all the systems and themes.

    Each item in the returned list is dict object containing the following key-value pairs.
    {
        "sass_source_dir": "",  # directory where source sass files are present
        "css_destination_dir": "",  # destination where css files would be placed
        "lookup_paths": [],  # list of directories to be passed as lookup paths for @import resolution.
    }
    """
    applicable_directories = []

    # add common sass directories
    applicable_directories.append({
        "sass_source_dir": path("common/static/sass"),
        "css_destination_dir": path("common/static/css"),
        "lookup_paths": COMMON_LOOKUP_PATHS,
    })

    return applicable_directories


def get_theme_sass_dirs(system, theme_dir):
    """
    Return list of sass dirs that need to be compiled for the given theme.

    :param system: name if the system for which to compile sass e.g. 'lms', 'cms'
    :param theme_dir: absolute path of theme for which to compile sass files.
    """
    if system not in ('lms', 'cms'):
        raise ValueError('"system" must either be "lms" or "cms"')

    dirs = []

    system_sass_dir = path(system) / "static" / "sass"
    sass_dir = theme_dir / system / "static" / "sass"
    css_dir = theme_dir / system / "static" / "css"
    certs_sass_dir = theme_dir / system / "static" / "certificates" / "sass"
    certs_css_dir = theme_dir / system / "static" / "certificates" / "css"

    dependencies = SASS_LOOKUP_DEPENDENCIES.get(system, [])
    if sass_dir.isdir():
        css_dir.mkdir_p()

        # first compile lms sass files and place css in theme dir
        dirs.append({
            "sass_source_dir": system_sass_dir,
            "css_destination_dir": css_dir,
            "lookup_paths": dependencies + [
                sass_dir / "partials",
                system_sass_dir / "partials",
                system_sass_dir,
            ],
        })

        # now compile theme sass files and override css files generated from lms
        dirs.append({
            "sass_source_dir": sass_dir,
            "css_destination_dir": css_dir,
            "lookup_paths": dependencies + [
                sass_dir / "partials",
                system_sass_dir / "partials",
                system_sass_dir,
            ],
        })

        # now compile theme sass files for certificate
        if system == 'lms':
            dirs.append({
                "sass_source_dir": certs_sass_dir,
                "css_destination_dir": certs_css_dir,
                "lookup_paths": [
                    sass_dir / "partials",
                    sass_dir
                ],
            })

    return dirs


def get_system_sass_dirs(system):
    """
    Return list of sass dirs that need to be compiled for the given system.

    :param system: name if the system for which to compile sass e.g. 'lms', 'cms'
    """
    if system not in ('lms', 'cms'):
        raise ValueError('"system" must either be "lms" or "cms"')

    dirs = []
    sass_dir = path(system) / "static" / "sass"
    css_dir = path(system) / "static" / "css"

    dependencies = SASS_LOOKUP_DEPENDENCIES.get(system, [])
    dirs.append({
        "sass_source_dir": sass_dir,
        "css_destination_dir": css_dir,
        "lookup_paths": dependencies + [
            sass_dir / "partials",
            sass_dir,
        ],
    })

    if system == 'lms':
        dirs.append({
            "sass_source_dir": path(system) / "static" / "certificates" / "sass",
            "css_destination_dir": path(system) / "static" / "certificates" / "css",
            "lookup_paths": [
                sass_dir / "partials",
                sass_dir
            ],
        })

    return dirs


def get_watcher_dirs(theme_dirs=None, themes=None):
    """
    Return sass directories that need to be added to sass watcher.

    Example:
        >> get_watcher_dirs('/edx/app/edx-platform/themes', ['red-theme'])
        [
            'common/static',
            'common/static/sass',
            'lms/static/sass',
            'lms/static/sass/partials',
            '/edx/app/edxapp/edx-platform/themes/red-theme/lms/static/sass',
            '/edx/app/edxapp/edx-platform/themes/red-theme/lms/static/sass/partials',
            'cms/static/sass',
            'cms/static/sass/partials',
            '/edx/app/edxapp/edx-platform/themes/red-theme/cms/static/sass/partials',
        ]

    Parameters:
        theme_dirs (list): list of theme base directories.
        themes (list): list containing names of themes
    Returns:
        (list): dirs that need to be added to sass watchers.
    """
    dirs = []
    dirs.extend(COMMON_LOOKUP_PATHS)
    if theme_dirs and themes:
        # Register sass watchers for all the given themes
        themes = get_theme_paths(themes=themes, theme_dirs=theme_dirs)
        for theme in themes:
            for _dir in get_sass_directories('lms', theme) + get_sass_directories('cms', theme):
                dirs.append(_dir['sass_source_dir'])
                dirs.extend(_dir['lookup_paths'])

    # Register sass watchers for lms and cms
    for _dir in get_sass_directories('lms') + get_sass_directories('cms') + get_common_sass_directories():
        dirs.append(_dir['sass_source_dir'])
        dirs.extend(_dir['lookup_paths'])

    # remove duplicates
    dirs = list(set(dirs))
    return dirs


def debounce(seconds=1):
    """
    Prevents the decorated function from being called more than every `seconds`
    seconds. Waits until calls stop coming in before calling the decorated
    function.
    """
    def decorator(func):
        func.timer = None

        @wraps(func)
        def wrapper(*args, **kwargs):
            def call():
                func(*args, **kwargs)
                func.timer = None
            if func.timer:
                func.timer.cancel()
            func.timer = Timer(seconds, call)
            func.timer.start()

        return wrapper
    return decorator


class SassWatcher(PatternMatchingEventHandler):
    """
    Watches for sass file changes
    """
    ignore_directories = True
    patterns = ['*.scss']
    ignore_patterns = ['common/static/xmodule/*']

    def register(self, observer, directories):
        """
        register files with observer

        Arguments:
            observer (watchdog.observers.Observer): sass file observer
            directories (list): list of directories to be register for sass watcher.
        """
        for dirname in directories:
            paths = []
            if '*' in dirname:
                paths.extend(glob.glob(dirname))
            else:
                paths.append(dirname)

            for obs_dirname in paths:
                observer.schedule(self, obs_dirname, recursive=True)

    @debounce()
    def on_any_event(self, event):
        print('\tCHANGED:', event.src_path)
        try:
            compile_sass()      # pylint: disable=no-value-for-parameter
        except Exception:       # pylint: disable=broad-except
            traceback.print_exc()


class XModuleSassWatcher(SassWatcher):
    """
    Watches for sass file changes
    """
    ignore_directories = True
    ignore_patterns = []

    @debounce()
    def on_any_event(self, event):
        print('\tCHANGED:', event.src_path)
        try:
            process_xmodule_assets()
        except Exception:  # pylint: disable=broad-except
            traceback.print_exc()


class XModuleAssetsWatcher(PatternMatchingEventHandler):
    """
    Watches for css and js file changes
    """
    ignore_directories = True
    patterns = ['*.css', '*.js']

    def register(self, observer):
        """
        Register files with observer
        """
        observer.schedule(self, 'xmodule/', recursive=True)

    @debounce()
    def on_any_event(self, event):
        print('\tCHANGED:', event.src_path)
        try:
            process_xmodule_assets()
        except Exception:  # pylint: disable=broad-except
            traceback.print_exc()

        # To refresh the hash values of static xmodule content
        restart_django_servers()


@task
@no_help
@cmdopts([
    ('system=', 's', 'The system to compile sass for (defaults to all)'),
    ('theme-dirs=', '-td', 'Theme dirs containing all themes (defaults to None)'),
    ('themes=', '-t', 'The theme to compile sass for (defaults to None)'),
    ('debug', 'd', 'Debug mode'),
    ('force', '', 'Force full compilation'),
])
@timed
def compile_sass(options):
    """
    Compile Sass to CSS. If command is called without any arguments, it will
    only compile lms, cms sass for the open source theme. And none of the comprehensive theme's sass would be compiled.

    If you want to compile sass for all comprehensive themes you will have to run compile_sass
    specifying all the themes that need to be compiled..

    The following is a list of some possible ways to use this command.

    Command:
        paver compile_sass
    Description:
        compile sass files for both lms and cms. If command is called like above (i.e. without any arguments) it will
        only compile lms, cms sass for the open source theme. None of the theme's sass will be compiled.

    Command:
        paver compile_sass --theme-dirs /edx/app/edxapp/edx-platform/themes --themes=red-theme
    Description:
        compile sass files for both lms and cms for 'red-theme' present in '/edx/app/edxapp/edx-platform/themes'

    Command:
        paver compile_sass --theme-dirs=/edx/app/edxapp/edx-platform/themes --themes red-theme stanford-style
    Description:
        compile sass files for both lms and cms for 'red-theme' and 'stanford-style' present in
        '/edx/app/edxapp/edx-platform/themes'.

    Command:
        paver compile_sass --system=cms
            --theme-dirs /edx/app/edxapp/edx-platform/themes /edx/app/edxapp/edx-platform/common/test/
            --themes red-theme stanford-style test-theme
    Description:
        compile sass files for cms only for 'red-theme', 'stanford-style' and 'test-theme' present in
        '/edx/app/edxapp/edx-platform/themes' and '/edx/app/edxapp/edx-platform/common/test/'.

    """
    debug = options.get('debug')
    force = options.get('force')
    systems = get_parsed_option(options, 'system', ALL_SYSTEMS)
    themes = get_parsed_option(options, 'themes', [])
    theme_dirs = get_parsed_option(options, 'theme_dirs', [])

    if not theme_dirs and themes:
        # We can not compile a theme sass without knowing the directory that contains the theme.
        raise ValueError('theme-dirs must be provided for compiling theme sass.')

    if themes and theme_dirs:
        themes = get_theme_paths(themes=themes, theme_dirs=theme_dirs)

    # Compile sass for OpenEdx theme after comprehensive themes
    if None not in themes:
        themes.append(None)

    timing_info = []
    dry_run = tasks.environment.dry_run
    compilation_results = {'success': [], 'failure': []}

    print("\t\tStarted compiling Sass:")

    # compile common sass files
    is_successful = _compile_sass('common', None, debug, force, timing_info)
    if is_successful:
        print("Finished compiling 'common' sass.")
    compilation_results['success' if is_successful else 'failure'].append('"common" sass files.')

    for system in systems:
        for theme in themes:
            print("Started compiling '{system}' Sass for '{theme}'.".format(system=system, theme=theme or 'system'))

            # Compile sass files
            is_successful = _compile_sass(
                system=system,
                theme=path(theme) if theme else None,
                debug=debug,
                force=force,
                timing_info=timing_info
            )

            if is_successful:
                print("Finished compiling '{system}' Sass for '{theme}'.".format(
                    system=system, theme=theme or 'system'
                ))

            compilation_results['success' if is_successful else 'failure'].append('{system} sass for {theme}.'.format(
                system=system, theme=theme or 'system',
            ))

    print("\t\tFinished compiling Sass:")
    if not dry_run:
        for sass_dir, css_dir, duration in timing_info:
            print(f">> {sass_dir} -> {css_dir} in {duration}s")

    if compilation_results['success']:
        print("\033[92m\nSuccessful compilations:\n--- " + "\n--- ".join(compilation_results['success']) + "\n\033[00m")
    if compilation_results['failure']:
        print("\033[91m\nFailed compilations:\n--- " + "\n--- ".join(compilation_results['failure']) + "\n\033[00m")


def _compile_sass(system, theme, debug, force, timing_info):
    """
    Compile sass files for the given system and theme.

    :param system: system to compile sass for e.g. 'lms', 'cms', 'common'
    :param theme: absolute path of the theme to compile sass for.
    :param debug: boolean showing whether to display source comments in resulted css
    :param force: boolean showing whether to remove existing css files before generating new files
    :param timing_info: list variable to keep track of timing for sass compilation
    """

    # Note: import sass only when it is needed and not at the top of the file.
    # This allows other paver commands to operate even without libsass being
    # installed. In particular, this allows the install_prereqs command to be
    # used to install the dependency.
    import sass
    if system == "common":
        sass_dirs = get_common_sass_directories()
    else:
        sass_dirs = get_sass_directories(system, theme)

    dry_run = tasks.environment.dry_run

    # determine css out put style and source comments enabling
    if debug:
        source_comments = True
        output_style = 'nested'
    else:
        source_comments = False
        output_style = 'compressed'

    for dirs in sass_dirs:
        start = datetime.now()
        css_dir = dirs['css_destination_dir']
        sass_source_dir = dirs['sass_source_dir']
        lookup_paths = dirs['lookup_paths']

        if not sass_source_dir.isdir():
            print("\033[91m Sass dir '{dir}' does not exists, skipping sass compilation for '{theme}' \033[00m".format(
                dir=sass_source_dir, theme=theme or system,
            ))
            # theme doesn't override sass directory, so skip it
            continue

        if force:
            if dry_run:
                tasks.environment.info("rm -rf {css_dir}/*.css".format(
                    css_dir=css_dir,
                ))
            else:
                sh(f"rm -rf {css_dir}/*.css")

        if dry_run:
            tasks.environment.info("libsass {sass_dir}".format(
                sass_dir=sass_source_dir,
            ))
        else:
            sass.compile(
                dirname=(sass_source_dir, css_dir),
                include_paths=COMMON_LOOKUP_PATHS + lookup_paths,
                source_comments=source_comments,
                output_style=output_style,
            )

        # For Sass files without explicit RTL versions, generate
        # an RTL version of the CSS using the rtlcss library.
        for sass_file in glob.glob(sass_source_dir + '/**/*.scss'):
            if should_generate_rtl_css_file(sass_file):
                source_css_file = sass_file.replace(sass_source_dir, css_dir).replace('.scss', '.css')
                target_css_file = source_css_file.replace('.css', '-rtl.css')
                sh("rtlcss {source_file} {target_file}".format(
                    source_file=source_css_file,
                    target_file=target_css_file,
                ))

        # Capture the time taken
        if not dry_run:
            duration = datetime.now() - start
            timing_info.append((sass_source_dir, css_dir, duration))
    return True


def should_generate_rtl_css_file(sass_file):
    """
    Returns true if a Sass file should have an RTL version generated.
    """
    # Don't generate RTL CSS for partials
    if path(sass_file).name.startswith('_'):
        return False

    # Don't generate RTL CSS if the file is itself an RTL version
    if sass_file.endswith('-rtl.scss'):
        return False

    # Don't generate RTL CSS if there is an explicit Sass version for RTL
    rtl_sass_file = path(sass_file.replace('.scss', '-rtl.scss'))
    if rtl_sass_file.exists():
        return False

    return True


def process_npm_assets():
    """
    Process vendor libraries installed via NPM.
    """
    def copy_vendor_library(library, skip_if_missing=False):
        """
        Copies a vendor library to the shared vendor directory.
        """
        if library.startswith('node_modules/'):
            library_path = library
        else:
            library_path = f'node_modules/{library}'

        if library.endswith('.css') or library.endswith('.css.map'):
            vendor_dir = NPM_CSS_VENDOR_DIRECTORY
        else:
            vendor_dir = NPM_JS_VENDOR_DIRECTORY
        if os.path.exists(library_path):
            sh('/bin/cp -rf {library_path} {vendor_dir}'.format(
                library_path=library_path,
                vendor_dir=vendor_dir,
            ))
        elif not skip_if_missing:
            raise Exception(f'Missing vendor file {library_path}')

    def copy_vendor_library_dir(library_dir, skip_if_missing=False):
        """
        Copies all vendor libraries in directory to the shared vendor directory.
        """
        library_dir_path = f'node_modules/{library_dir}'
        print(f'Copying vendor library dir: {library_dir_path}')
        if os.path.exists(library_dir_path):
            for dirpath, _, filenames in os.walk(library_dir_path):
                for filename in filenames:
                    copy_vendor_library(os.path.join(dirpath, filename), skip_if_missing=skip_if_missing)

    # Skip processing of the libraries if this is just a dry run
    if tasks.environment.dry_run:
        tasks.environment.info("install npm_assets")
        return

    # Ensure that the vendor directory exists
    NPM_JS_VENDOR_DIRECTORY.mkdir_p()
    NPM_CSS_DIRECTORY.mkdir_p()
    NPM_CSS_VENDOR_DIRECTORY.mkdir_p()

    # Copy each file to the vendor directory, overwriting any existing file.
    print("Copying vendor files into static directory")
    for library in NPM_INSTALLED_LIBRARIES:
        if library.endswith('/'):
            copy_vendor_library_dir(library)
        else:
            copy_vendor_library(library)

    # Copy over each developer library too if they have been installed
    print("Copying developer vendor files into static directory")
    for library in NPM_INSTALLED_DEVELOPER_LIBRARIES:
        copy_vendor_library(library, skip_if_missing=True)


@task
@needs(
    'pavelib.prereqs.install_python_prereqs',
)
@no_help
def process_xmodule_assets():
    """
    Process XModule static assets.
    """
    sh('xmodule_assets common/static/xmodule')
    print("\t\tFinished processing xmodule assets.")


def restart_django_servers():
    """
    Restart the django server.

    `$ touch` makes the Django file watcher thinks that something has changed, therefore
    it restarts the server.
    """
    sh(cmd(
        "touch", 'lms/urls.py', 'cms/urls.py',
    ))


def collect_assets(systems, settings, **kwargs):
    """
    Collect static assets, including Django pipeline processing.
    `systems` is a list of systems (e.g. 'lms' or 'studio' or both)
    `settings` is the Django settings module to use.
    `**kwargs` include arguments for using a log directory for collectstatic output. Defaults to /dev/null.
    """
    ignore_patterns = [
        # Karma test related files...
        "fixtures",
        "karma_*.js",
        "spec",
        "spec_helpers",
        "spec-helpers",
        "xmodule_js",  # symlink for tests

        # Geo-IP data, only accessed in Python
        "geoip",

        # We compile these out, don't need the source files in staticfiles
        "sass",
    ]

    ignore_args = " ".join(
        f'--ignore "{pattern}"' for pattern in ignore_patterns
    )

    for sys in systems:
        collectstatic_stdout_str = _collect_assets_cmd(sys, **kwargs)
        sh(django_cmd(sys, settings, "collectstatic {ignore_args} --noinput {logfile_str}".format(
            ignore_args=ignore_args,
            logfile_str=collectstatic_stdout_str
        )))
        print(f"\t\tFinished collecting {sys} assets.")


def _collect_assets_cmd(system, **kwargs):
    """
    Returns the collecstatic command to be used for the given system

    Unless specified, collectstatic (which can be verbose) pipes to /dev/null
    """
    try:
        if kwargs[COLLECTSTATIC_LOG_DIR_ARG] is None:
            collectstatic_stdout_str = ""
        else:
            collectstatic_stdout_str = "> {output_dir}/{sys}-collectstatic.log".format(
                output_dir=kwargs[COLLECTSTATIC_LOG_DIR_ARG],
                sys=system
            )
    except KeyError:
        collectstatic_stdout_str = "> /dev/null"

    return collectstatic_stdout_str


def execute_compile_sass(args):
    """
    Construct django management command compile_sass (defined in theming app) and execute it.
    Args:
        args: command line argument passed via update_assets command
    """
    for sys in args.system:
        options = ""
        options += " --theme-dirs " + " ".join(args.theme_dirs) if args.theme_dirs else ""
        options += " --themes " + " ".join(args.themes) if args.themes else ""
        options += " --debug" if args.debug else ""

        sh(
            django_cmd(
                sys,
                args.settings,
                "compile_sass {system} {options}".format(
                    system='cms' if sys == 'studio' else sys,
                    options=options,
                ),
            ),
        )


@task
@cmdopts([
    ('settings=', 's', "Django settings (defaults to devstack)"),
    ('watch', 'w', "Watch file system and rebuild on change (defaults to off)"),
])
@timed
def webpack(options):
    """
    Run a Webpack build.
    """
    settings = getattr(options, 'settings', Env.DEVSTACK_SETTINGS)
    result = Env.get_django_settings(['STATIC_ROOT', 'WEBPACK_CONFIG_PATH'], "lms", settings=settings)
    static_root_lms, config_path = result
    static_root_cms, = Env.get_django_settings(["STATIC_ROOT"], "cms", settings=settings)
    js_env_extra_config_setting, = Env.get_django_json_settings(["JS_ENV_EXTRA_CONFIG"], "cms", settings=settings)
    js_env_extra_config = json.dumps(js_env_extra_config_setting or "{}")
    environment = (
        "NODE_ENV={node_env} STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} "
        "JS_ENV_EXTRA_CONFIG={js_env_extra_config}"
    ).format(
        node_env="development" if config_path == 'webpack.dev.config.js' else "production",
        static_root_lms=static_root_lms,
        static_root_cms=static_root_cms,
        js_env_extra_config=js_env_extra_config,
    )
    sh(
        cmd(
            '{environment} $(npm bin)/webpack --config={config_path}'.format(
                environment=environment,
                config_path=config_path
            )
        )
    )


def execute_webpack_watch(settings=None):
    """
    Run the Webpack file system watcher.
    """
    # We only want Webpack to re-run on changes to its own entry points,
    # not all JS files, so we use its own watcher instead of subclassing
    # from Watchdog like the other watchers do.

    result = Env.get_django_settings(["STATIC_ROOT", "WEBPACK_CONFIG_PATH"], "lms", settings=settings)
    static_root_lms, config_path = result
    static_root_cms, = Env.get_django_settings(["STATIC_ROOT"], "cms", settings=settings)
    run_background_process(
        'STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} $(npm bin)/webpack {options}'.format(
            options='--watch --config={config_path}'.format(
                config_path=config_path
            ),
            static_root_lms=static_root_lms,
            static_root_cms=static_root_cms,
        )
    )


def get_parsed_option(command_opts, opt_key, default=None):
    """
    Extract user command option and parse it.
    Arguments:
        command_opts: Command line arguments passed via paver command.
        opt_key: name of option to get and parse
        default: if `command_opt_value` not in `command_opts`, `command_opt_value` will be set to default.
    Returns:
         list or None
    """
    command_opt_value = getattr(command_opts, opt_key, default)
    if command_opt_value:
        command_opt_value = listfy(command_opt_value)

    return command_opt_value


def listfy(data):
    """
    Check and convert data to list.
    Arguments:
        data: data structure to be converted.
    """

    if isinstance(data, str):
        data = data.split(',')
    elif not isinstance(data, list):
        data = [data]

    return data


@task
@cmdopts([
    ('background', 'b', 'Background mode'),
    ('settings=', 's', "Django settings (defaults to devstack)"),
    ('theme-dirs=', '-td', 'The themes dir containing all themes (defaults to None)'),
    ('themes=', '-t', 'The themes to add sass watchers for (defaults to None)'),
    ('wait=', '-w', 'How long to pause between filesystem scans.')
])
@timed
def watch_assets(options):
    """
    Watch for changes to asset files, and regenerate js/css
    """
    # Don't watch assets when performing a dry run
    if tasks.environment.dry_run:
        return

    settings = getattr(options, 'settings', Env.DEVSTACK_SETTINGS)

    themes = get_parsed_option(options, 'themes')
    theme_dirs = get_parsed_option(options, 'theme_dirs', [])

    default_wait = [str(DEFAULT_OBSERVER_TIMEOUT)]
    wait = float(get_parsed_option(options, 'wait', default_wait)[0])

    if not theme_dirs and themes:  # lint-amnesty, pylint: disable=no-else-raise
        # We can not add theme sass watchers without knowing the directory that contains the themes.
        raise ValueError('theme-dirs must be provided for watching theme sass.')
    else:
        theme_dirs = [path(_dir) for _dir in theme_dirs]

    sass_directories = get_watcher_dirs(theme_dirs, themes)
    observer = Observer(timeout=wait)

    SassWatcher().register(observer, sass_directories)
    XModuleSassWatcher().register(observer, ['xmodule/'])
    XModuleAssetsWatcher().register(observer)

    print("Starting asset watcher...")
    observer.start()

    # Run the Webpack file system watcher too
    execute_webpack_watch(settings=settings)

    if not getattr(options, 'background', False):
        # when running as a separate process, the main thread needs to loop
        # in order to allow for shutdown by control-c
        try:
            while True:
                observer.join(2)
        except KeyboardInterrupt:
            observer.stop()
        print("\nStopped asset watcher.")


@task
@needs(
    'pavelib.prereqs.install_node_prereqs',
)
@consume_args
@timed
def update_assets(args):
    """
    Compile Sass, then collect static assets.
    """
    parser = argparse.ArgumentParser(prog='paver update_assets')
    parser.add_argument(
        'system', type=str, nargs='*', default=ALL_SYSTEMS,
        help="lms or studio",
    )
    parser.add_argument(
        '--settings', type=str, default=Env.DEVSTACK_SETTINGS,
        help="Django settings module",
    )
    parser.add_argument(
        '--debug', action='store_true', default=False,
        help="Enable all debugging",
    )
    parser.add_argument(
        '--debug-collect', action='store_true', default=False,
        help="Disable collect static",
    )
    parser.add_argument(
        '--skip-collect', dest='collect', action='store_false', default=True,
        help="Skip collection of static assets",
    )
    parser.add_argument(
        '--watch', action='store_true', default=False,
        help="Watch files for changes",
    )
    parser.add_argument(
        '--theme-dirs', dest='theme_dirs', type=str, nargs='+', default=None,
        help="base directories where themes are placed",
    )
    parser.add_argument(
        '--themes', type=str, nargs='+', default=None,
        help="list of themes to compile sass for",
    )
    parser.add_argument(
        '--collect-log', dest=COLLECTSTATIC_LOG_DIR_ARG, default=None,
        help="When running collectstatic, direct output to specified log directory",
    )
    parser.add_argument(
        '--wait', type=float, default=0.0,
        help="How long to pause between filesystem scans"
    )
    args = parser.parse_args(args)
    collect_log_args = {}

    process_xmodule_assets()
    process_npm_assets()

    # Build Webpack
    call_task('pavelib.assets.webpack', options={'settings': args.settings})

    # Compile sass for themes and system
    execute_compile_sass(args)

    if args.collect:
        if args.debug or args.debug_collect:
            collect_log_args.update({COLLECTSTATIC_LOG_DIR_ARG: None})

        if args.collect_log_dir:
            collect_log_args.update({COLLECTSTATIC_LOG_DIR_ARG: args.collect_log_dir})

        collect_assets(args.system, args.settings, **collect_log_args)

    if args.watch:
        call_task(
            'pavelib.assets.watch_assets',
            options={
                'background': not args.debug,
                'settings': args.settings,
                'theme_dirs': args.theme_dirs,
                'themes': args.themes,
                'wait': [float(args.wait)]
            },
        )