Skip to content

Commit

Permalink
Make artifact tests not flaky
Browse files Browse the repository at this point in the history
Previously some of the artifact tests would fail, if the minute the assertion happened was different
from the minute the artifact was created.
Now we verify they happened within a minute of each other, so
this should be more robust.
  • Loading branch information
saulshanabrook committed Mar 31, 2022
1 parent ae7d653 commit 1ffd81d
Showing 1 changed file with 11 additions and 6 deletions.
17 changes: 11 additions & 6 deletions tests/end_to_end/test_artifacts.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from datetime import datetime
from datetime import datetime, timedelta

from lineapy.utils.constants import VERSION_DATE_STRING

Expand Down Expand Up @@ -71,10 +71,11 @@ def test_get_artifact_has_version(execute):
art_version = art.version
"""
res = execute(code, snapshot=False)
# doing this because if the test runs at the edge of the second it fails sometimes
assert res.values["art_version"].startswith(
datetime.now().strftime("%Y-%m-%dT%H:%M:")
# Verify the version date is at most a minute from now but not in the future
artifact_version_delta = datetime.now() - datetime.fromisoformat(
res.values["art_version"]
)
assert timedelta(minutes=0) < artifact_version_delta < timedelta(minutes=1)
assert res.slice("x") == "x = 1\n"


Expand Down Expand Up @@ -107,9 +108,13 @@ def test_catalog_shows_all_versions(execute):
# artifact_version and date_created might not match esp in future with named versions
# but for default version it should be a string version of a date
assert db_values[0]["date_created"] < db_values[1]["date_created"]
assert db_values[1]["artifact_version"].startswith(
datetime.now().strftime("%Y-%m-%dT%H:%M:")

# Verify the version date is at most a minute old but not in the future
artifact_version_delta = datetime.now() - datetime.fromisoformat(
db_values[1]["artifact_version"]
)
assert timedelta(minutes=0) < artifact_version_delta < timedelta(minutes=1)

# finally make sure the print property is updated to reflect versions
assert res.values["all_print"] == "\n".join(
[
Expand Down

0 comments on commit 1ffd81d

Please sign in to comment.