test-js-with-node

#!/usr/bin/env python3
import argparse
import glob
import os
import pwd
import subprocess
import sys
from typing import Any, Dict, List, Set

TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.dirname(TOOLS_DIR))
ROOT_DIR = os.path.dirname(TOOLS_DIR)

# check for the venv
from tools.lib import sanity_check

sanity_check.check_venv(__file__)

# Import this after we do the sanity_check so it doesn't crash.
import orjson
from zulint.printer import BOLDRED, CYAN, ENDC, GREEN

INDEX_JS = os.path.join(ROOT_DIR, "web/tests/lib/index.js")
NODE_COVERAGE_PATH = os.path.join(ROOT_DIR, "var/node-coverage/coverage-final.json")

# Ideally, we wouldn't need this line, but it seems to be required to
# avoid problems finding node_modules when running `cd tools; ./test-js-with-node`.
os.chdir(ROOT_DIR)

USAGE = """
    tools/test-js-with-node                                - to run all tests
    tools/test-js-with-node util.test.js activity.test.js  - to run just a couple tests
    tools/test-js-with-node --coverage                     - to generate coverage report
    """


def make_set(files: List[str]) -> Set[str]:
    for i in range(1, len(files)):
        if files[i - 1] > files[i]:
            raise Exception(f"Please move {files[i]} so that names are sorted.")
    return set(files)


# We do not yet require 100% line coverage for these files:
EXEMPT_FILES = make_set(
    [
        "web/shared/src/poll_data.ts",
        "web/src/about_zulip.ts",
        "web/src/add_stream_options_popover.ts",
        "web/src/add_subscribers_pill.js",
        "web/src/admin.js",
        "web/src/alert_popup.ts",
        "web/src/alert_words_ui.ts",
        "web/src/archive.js",
        "web/src/assets.d.ts",
        "web/src/attachments_ui.ts",
        "web/src/audible_notifications.ts",
        "web/src/avatar.ts",
        "web/src/base_page_params.ts",
        "web/src/billing/event_status.ts",
        "web/src/billing/helpers.ts",
        "web/src/blueslip.ts",
        "web/src/blueslip_stacktrace.ts",
        "web/src/bootstrap_typeahead.ts",
        "web/src/browser_history.ts",
        "web/src/buddy_list.ts",
        "web/src/click_handlers.js",
        "web/src/compose.js",
        "web/src/compose_actions.ts",
        "web/src/compose_banner.ts",
        "web/src/compose_call_ui.js",
        "web/src/compose_closed_ui.js",
        "web/src/compose_fade.ts",
        "web/src/compose_notifications.ts",
        "web/src/compose_popovers.js",
        "web/src/compose_recipient.ts",
        "web/src/compose_reply.js",
        "web/src/compose_send_menu_popover.js",
        "web/src/compose_setup.js",
        "web/src/compose_state.ts",
        "web/src/compose_textarea.ts",
        "web/src/compose_tooltips.ts",
        "web/src/compose_ui.ts",
        "web/src/compose_validate.ts",
        "web/src/composebox_typeahead.js",
        "web/src/condense.ts",
        "web/src/confirm_dialog.ts",
        "web/src/copied_tooltip.ts",
        "web/src/copy_and_paste.js",
        "web/src/csrf.ts",
        "web/src/css_variables.d.ts",
        "web/src/css_variables.js",
        "web/src/custom_profile_fields_ui.js",
        "web/src/dark_theme.ts",
        "web/src/debug.ts",
        "web/src/demo_organizations_ui.js",
        "web/src/deprecated_feature_notice.ts",
        "web/src/desktop_integration.js",
        "web/src/desktop_notifications.ts",
        "web/src/dialog_widget.ts",
        "web/src/drafts.ts",
        "web/src/drafts_overlay_ui.js",
        "web/src/dropdown_widget.ts",
        "web/src/echo.js",
        "web/src/electron_bridge.d.ts",
        "web/src/email_pill.ts",
        "web/src/emoji_picker.js",
        "web/src/emojisets.ts",
        "web/src/favicon.ts",
        "web/src/feedback_widget.ts",
        "web/src/fetch_status.ts",
        "web/src/flatpickr.ts",
        "web/src/gear_menu.js",
        "web/src/giphy.js",
        "web/src/global.ts",
        "web/src/hash_util.ts",
        "web/src/hashchange.js",
        "web/src/hbs.d.ts",
        "web/src/hotkey.js",
        "web/src/hotspots.ts",
        "web/src/inbox_ui.js",
        "web/src/inbox_util.ts",
        "web/src/info_overlay.ts",
        "web/src/information_density.ts",
        "web/src/integration_url_modal.ts",
        "web/src/invite.ts",
        "web/src/left_sidebar_navigation_area.ts",
        "web/src/left_sidebar_navigation_area_popovers.js",
        "web/src/lightbox.ts",
        "web/src/list_util.ts",
        "web/src/list_widget.ts",
        "web/src/loading.ts",
        "web/src/local_message.ts",
        "web/src/localstorage.ts",
        "web/src/message_actions_popover.js",
        "web/src/message_edit.js",
        "web/src/message_edit_history.ts",
        "web/src/message_events.js",
        "web/src/message_feed_loading.ts",
        "web/src/message_feed_top_notices.ts",
        "web/src/message_fetch.js",
        "web/src/message_list.js",
        "web/src/message_list_data.ts",
        "web/src/message_list_hover.js",
        "web/src/message_list_tooltips.ts",
        "web/src/message_list_view.js",
        "web/src/message_lists.ts",
        "web/src/message_live_update.ts",
        "web/src/message_notifications.js",
        "web/src/message_scroll.js",
        "web/src/message_scroll_state.ts",
        "web/src/message_util.ts",
        "web/src/message_view_header.ts",
        "web/src/message_viewport.ts",
        "web/src/messages_overlay_ui.ts",
        "web/src/modals.ts",
        "web/src/muted_users_ui.js",
        "web/src/narrow.js",
        "web/src/narrow_history.ts",
        "web/src/narrow_title.ts",
        "web/src/navbar_alerts.js",
        "web/src/navbar_help_menu.ts",
        "web/src/navbar_menus.js",
        "web/src/navigate.js",
        "web/src/onboarding_steps.ts",
        "web/src/overlay_util.ts",
        "web/src/overlays.ts",
        "web/src/padded_widget.ts",
        "web/src/page_params.ts",
        "web/src/personal_menu_popover.js",
        "web/src/playground_links_popover.ts",
        "web/src/plotly.js.d.ts",
        "web/src/pm_list.ts",
        "web/src/pm_list_dom.ts",
        "web/src/poll_modal.ts",
        "web/src/poll_widget.ts",
        "web/src/popover_menus.ts",
        "web/src/popover_menus_data.js",
        "web/src/popovers.ts",
        "web/src/popperjs.d.ts",
        "web/src/read_receipts.ts",
        "web/src/ready.ts",
        "web/src/realm_icon.ts",
        "web/src/realm_logo.ts",
        "web/src/realm_playground.ts",
        "web/src/realm_user_settings_defaults.ts",
        "web/src/recent_view_ui.js",
        "web/src/reload.js",
        "web/src/reload_setup.js",
        "web/src/reminder.js",
        "web/src/resize.ts",
        "web/src/resize_handler.ts",
        "web/src/rows.ts",
        "web/src/scheduled_messages.ts",
        "web/src/scheduled_messages_feed_ui.ts",
        "web/src/scheduled_messages_overlay_ui.js",
        "web/src/scheduled_messages_ui.js",
        "web/src/scroll_bar.ts",
        "web/src/scroll_util.ts",
        "web/src/search.ts",
        "web/src/sent_messages.ts",
        "web/src/sentry.ts",
        "web/src/server_events.js",
        "web/src/settings.js",
        "web/src/settings_account.js",
        "web/src/settings_bots.js",
        "web/src/settings_components.ts",
        "web/src/settings_emoji.ts",
        "web/src/settings_exports.ts",
        "web/src/settings_invites.ts",
        "web/src/settings_linkifiers.ts",
        "web/src/settings_muted_users.ts",
        "web/src/settings_notifications.js",
        "web/src/settings_org.js",
        "web/src/settings_panel_menu.js",
        "web/src/settings_playgrounds.ts",
        "web/src/settings_preferences.js",
        "web/src/settings_profile_fields.js",
        "web/src/settings_realm_domains.ts",
        "web/src/settings_realm_user_settings_defaults.js",
        "web/src/settings_sections.js",
        "web/src/settings_streams.js",
        "web/src/settings_toggle.js",
        "web/src/settings_ui.ts",
        "web/src/settings_user_topics.ts",
        "web/src/settings_users.js",
        "web/src/setup.ts",
        "web/src/sidebar_ui.ts",
        "web/src/spectators.ts",
        "web/src/spoilers.ts",
        "web/src/starred_messages_ui.js",
        "web/src/state_data.ts",
        "web/src/stream_color.ts",
        "web/src/stream_color_events.ts",
        "web/src/stream_create.js",
        "web/src/stream_create_subscribers.js",
        "web/src/stream_edit.js",
        "web/src/stream_edit_subscribers.js",
        "web/src/stream_edit_toggler.ts",
        "web/src/stream_list.ts",
        "web/src/stream_muting.js",
        "web/src/stream_popover.js",
        "web/src/stream_settings_api.ts",
        "web/src/stream_settings_components.js",
        "web/src/stream_settings_containers.ts",
        "web/src/stream_settings_ui.js",
        "web/src/stream_ui_updates.js",
        "web/src/submessage.ts",
        "web/src/subscriber_api.ts",
        "web/src/timerender.ts",
        "web/src/tippyjs.ts",
        "web/src/todo_widget.js",
        "web/src/topic_list.ts",
        "web/src/topic_popover.js",
        "web/src/tutorial.js",
        "web/src/types.ts",
        "web/src/typing.ts",
        "web/src/typing_events.ts",
        "web/src/ui_init.js",
        "web/src/ui_report.ts",
        "web/src/ui_util.ts",
        "web/src/unread.ts",
        "web/src/unread_ops.js",
        "web/src/unread_ui.ts",
        "web/src/upload.js",
        "web/src/upload_widget.ts",
        "web/src/url-template.d.ts",
        "web/src/user_card_popover.js",
        "web/src/user_deactivation_ui.ts",
        "web/src/user_events.js",
        "web/src/user_group_components.ts",
        "web/src/user_group_create.js",
        "web/src/user_group_create_members.js",
        "web/src/user_group_create_members_data.ts",
        "web/src/user_group_edit.js",
        "web/src/user_group_edit_members.js",
        "web/src/user_group_popover.ts",
        "web/src/user_group_ui_updates.js",
        "web/src/user_groups.ts",
        "web/src/user_pill.ts",
        "web/src/user_profile.js",
        "web/src/user_settings.ts",
        "web/src/user_sort.ts",
        "web/src/user_status.ts",
        "web/src/user_status_ui.ts",
        "web/src/user_topic_popover.js",
        "web/src/user_topics.ts",
        "web/src/user_topics_ui.js",
        "web/src/views_util.ts",
        "web/src/zcommand.ts",
        "web/src/zform.js",
        "web/src/zulip.js",
        "web/src/zulip_test.ts",
        "web/tests/lib/mdiff.js",
        "web/tests/lib/real_jquery.js",
        "web/tests/lib/zjquery_element.js",
        "web/tests/lib/zpage_billing_params.js",
        # There are some important functions which are not called right now but will
        # be reused when we add tests for dropdown widget so it doesn't make sense to remove them.
        "web/tests/recent_view.test.js",
    ]
)

from tools.lib.test_script import add_provision_check_override_param, assert_provisioning_status_ok

parser = argparse.ArgumentParser(USAGE)
parser.add_argument("--coverage", action="store_true", help="Get coverage report")
add_provision_check_override_param(parser)
parser.add_argument("args", nargs=argparse.REMAINDER)
parser.add_argument(
    "--parallel",
    dest="parallel",
    action="store",
    type=int,
    # Since process startup time is a significant portion of total
    # runtime, so rather than doing os.cpu_count, we just do a fixed 4
    # processes by default.
    default=4,
    help="Specify the number of processes to run the "
    "tests in. Default is the number of logical CPUs",
)
options = parser.parse_args()
individual_files = options.args
parallel = options.parallel

if options.coverage and parallel > 1:
    parallel = 1
    print(
        BOLDRED + "You cannot use --coverage with parallel tests. Running in serial mode.\n" + ENDC
    )

assert_provisioning_status_ok(options.skip_provision_check)


def get_dev_host() -> str:
    # See similar code in dev_settings.py.  We only use
    # this to report where you can find coverage reports.
    # We duplicate the code here to avoid depending on
    # Django.

    host = os.getenv("EXTERNAL_HOST")
    if host is not None:
        return host

    user_id = os.getuid()
    user_name = pwd.getpwuid(user_id).pw_name
    if user_name == "zulipdev":
        hostname = os.uname()[1].lower()
        if ".zulipdev.org" not in hostname:
            hostname += ".zulipdev.org"
        return hostname + ":9991"
    else:
        # For local development environments, we use localhost by
        # default, via the "zulipdev.com" hostname.
        return "zulipdev.com:9991"


def print_error(msg: str) -> None:
    print(BOLDRED + "ERROR:" + ENDC + " " + msg)


def clean_file(orig_fn: str) -> str:
    fn = orig_fn
    if not fn.endswith(".test.js"):
        fn += ".test.js"
    if "web/tests/" not in fn:
        fn = os.path.join(ROOT_DIR, "web", "tests", fn)
    fn = os.path.abspath(fn)
    if not os.path.exists(fn):
        print(f"Cannot find {orig_fn} ({fn})")
        sys.exit(1)
    return fn


def clean_files(fns: List[str]) -> List[str]:
    cleaned_files = [clean_file(fn) for fn in fns]
    return cleaned_files


def run_tests_via_node_js() -> int:
    os.environ["TZ"] = "UTC"

    # The index.js test runner is the real "driver" here, and we launch
    # with either nyc or node, depending on whether we want coverage
    # reports.  Running under nyc is slower and creates funny
    # tracebacks, so you generally want to get coverage reports only
    # after making sure tests will pass.
    node_tests_cmd = ["node", "--stack-trace-limit=100", INDEX_JS]
    if individual_files:
        # If we passed a specific set of tests, run in serial mode.
        global parallel
        parallel = 1
        files = individual_files
    else:
        files = sorted(glob.glob(os.path.join(ROOT_DIR, "web/tests/*.test.js")))

    test_files = clean_files(files)

    print("Starting node tests...")

    # If we got this far, we can run the tests!
    ret = 0
    if parallel > 1:
        sub_tests = [test_files[i::parallel] for i in range(parallel)]
        parallel_processes = [subprocess.Popen(node_tests_cmd + sub_test) for sub_test in sub_tests]

        for process in parallel_processes:
            status_code = process.wait()
            if status_code != 0:
                ret = status_code
        return ret

    node_tests_cmd += test_files
    if options.coverage:
        os.environ["USING_INSTRUMENTED_CODE"] = "TRUE"
        coverage_dir = os.path.join(ROOT_DIR, "var/node-coverage")

        nyc = os.path.join(ROOT_DIR, "node_modules/.bin/nyc")
        command = [nyc, "--extension", ".hbs", "--extension", ".ts"]
        command += ["--report-dir", coverage_dir]
        command += ["--temp-directory", coverage_dir]
        command += ["-r=lcov", "-r=json", "-r=text-summary"]
        command += node_tests_cmd
    else:
        # Normal testing, no coverage analysis.
        # Run the index.js test runner, which runs all the other tests.
        command = node_tests_cmd

    try:
        ret = subprocess.check_call(command)
    except OSError:
        print(f"Bad command: {command}")
        raise
    except subprocess.CalledProcessError:
        print("\n** Tests failed, PLEASE FIX! **\n")
        sys.exit(1)
    return ret


def check_line_coverage(
    fn: str, line_coverage: Dict[Any, Any], line_mapping: Dict[Any, Any], log: bool = True
) -> bool:
    missing_lines = []
    for line in line_coverage:
        if line_coverage[line] == 0:
            actual_line = line_mapping[line]
            missing_lines.append(str(actual_line["start"]["line"]))
    if missing_lines:
        if log:
            print_error(f"{fn} no longer has complete node test coverage")
            print("  Lines missing coverage: {}".format(", ".join(sorted(missing_lines, key=int))))
            print()
        return False
    return True


def read_coverage() -> Any:
    coverage_json = None
    try:
        with open(NODE_COVERAGE_PATH, "rb") as f:
            coverage_json = orjson.loads(f.read())
    except OSError:
        print(NODE_COVERAGE_PATH + " doesn't exist. Cannot enforce fully covered files.")
        raise
    return coverage_json


def enforce_proper_coverage(coverage_json: Any) -> bool:
    all_js_files = {
        *glob.glob("web/shared/src/*.js"),
        *glob.glob("web/shared/src/*.ts"),
        *glob.glob("web/src/*.js"),
        *glob.glob("web/src/*.ts"),
        *glob.glob("web/src/billing/*.js"),
        *glob.glob("web/tests/*.js"),
        *glob.glob("web/tests/lib/*.js"),
    }
    enforce_fully_covered = sorted(all_js_files - EXEMPT_FILES)

    coverage_lost = False
    for relative_path in enforce_fully_covered:
        path = ROOT_DIR + "/" + relative_path
        if path not in coverage_json:
            coverage_lost = True
            print_error(f"{relative_path} has no node test coverage")
            continue
        line_coverage = coverage_json[path]["s"]
        line_mapping = coverage_json[path]["statementMap"]
        if not check_line_coverage(relative_path, line_coverage, line_mapping):
            coverage_lost = True
    if coverage_lost:
        print()
        print("It looks like your changes lost 100% test coverage in one or more files.")
        print("Ideally, you should add some tests to restore coverage.")
        print("A worse option is to update EXEMPT_FILES in `tools/test-js-with-node`.")
        print("To run this check locally, use `test-js-with-node --coverage`.")
        print()

    coverage_not_enforced = False
    for path in coverage_json:
        relative_path = os.path.relpath(path, ROOT_DIR)
        if relative_path in EXEMPT_FILES:
            line_coverage = coverage_json[path]["s"]
            line_mapping = coverage_json[path]["statementMap"]
            if check_line_coverage(relative_path, line_coverage, line_mapping, log=False):
                coverage_not_enforced = True
                print_error(f"{relative_path} unexpectedly has 100% line coverage.")

    if coverage_not_enforced:
        print()
        print("One or more fully covered files are miscategorized.")
        print("Remove the file(s) from EXEMPT_FILES in `tools/test-js-with-node`.")

    problems_encountered = coverage_lost or coverage_not_enforced
    return problems_encountered


ret = run_tests_via_node_js()

if options.coverage and ret == 0:
    if not individual_files:
        coverage_json = read_coverage()
        problems_encountered = enforce_proper_coverage(coverage_json)
        if problems_encountered:
            ret = 1

    reports_location = f"http://{get_dev_host()}/node-coverage/index.html"
    print()
    print("View coverage reports at " + CYAN + reports_location + ENDC)

print()
if ret == 0:
    print(GREEN + "Test(s) passed. SUCCESS!" + ENDC)
else:
    print(BOLDRED + "FAIL - Test(s) failed" + ENDC)

sys.exit(ret)