-
Notifications
You must be signed in to change notification settings - Fork 24.7k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
EQL: Sequences will now support nano-timestamps #76953
Changes from 7 commits
763c61e
a4a5679
a6dd270
ad90d3b
2ba7386
b97d3ab
db31711
35b9d0c
ff31707
5bd470a
bc77ffb
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -6,6 +6,7 @@ | |
*/ | ||
package org.elasticsearch.test.eql; | ||
|
||
import static org.hamcrest.Matchers.greaterThan; | ||
import static org.hamcrest.Matchers.instanceOf; | ||
import static org.junit.Assert.assertThat; | ||
|
||
|
@@ -18,6 +19,7 @@ | |
import java.util.List; | ||
import java.util.Map; | ||
import java.util.Map.Entry; | ||
import java.util.function.Consumer; | ||
|
||
import org.apache.http.HttpHost; | ||
import org.apache.logging.log4j.LogManager; | ||
|
@@ -44,20 +46,21 @@ | |
* Loads EQL dataset into ES. | ||
* | ||
* Checks for predefined indices: | ||
* - endgame-140 - for existing data | ||
* - extra - additional data | ||
* - endgame-140 - for existing data | ||
* - endgame-140-nanos - same as endgame-140, but with nano-precision timestamps | ||
* - extra - additional data | ||
* | ||
* While the loader could be made generic, the queries are bound to each index and generalizing that would make things way too complicated. | ||
*/ | ||
public class DataLoader { | ||
public static final String TEST_INDEX = "endgame-140"; | ||
public static final String TEST_EXTRA_INDEX = "extra"; | ||
public static final String DATE_NANOS_INDEX = "eql_date_nanos"; | ||
public static final String TEST_NANOS_INDEX = "endgame-140-nanos"; | ||
|
||
private static final Map<String, String[]> replacementPatterns = Collections.unmodifiableMap(getReplacementPatterns()); | ||
|
||
private static final long FILETIME_EPOCH_DIFF = 11644473600000L; | ||
private static final long FILETIME_ONE_MILLISECOND = 10 * 1000; | ||
private static final long FILETIME_EPOCH_DIFF = 11644473600000L; // millis delta from the start of year 1601 (Windows filetime) to 1970 | ||
private static final long FILETIME_ONE_MILLISECOND = 10 * 1000; // Windows filetime is in 100-nanoseconds ticks | ||
|
||
// runs as java main | ||
private static boolean main = false; | ||
|
@@ -86,33 +89,34 @@ public static void loadDatasetIntoEs(RestHighLevelClient client, | |
// | ||
// Main Index | ||
// | ||
load(client, TEST_INDEX, true, p); | ||
load(client, TEST_INDEX, null, DataLoader::timestampToUnixMillis, p); | ||
// | ||
// Aux Index | ||
// | ||
load(client, TEST_EXTRA_INDEX, false, p); | ||
load(client, TEST_EXTRA_INDEX, null, null, p); | ||
// | ||
// Date_Nanos index | ||
// | ||
// The data for this index are identical to the endgame-140.data with only the values for @timestamp changed. | ||
// The data for this index is loaded from the same endgame-140.data sample, only having the mapping for @timestamp changed: the | ||
// chosen Windows filetime timestamps (2017+) can coincidentally also be readily used as nano-resolution unix timestamps (1973+). | ||
// There are mixed values with and without nanos precision so that the filtering is properly tested for both cases. | ||
load(client, DATE_NANOS_INDEX, false, p); | ||
load(client, TEST_NANOS_INDEX, TEST_INDEX, DataLoader::timestampToUnixNanos, p); | ||
} | ||
|
||
private static void load(RestHighLevelClient client, String indexName, boolean winFileTime, | ||
private static void load(RestHighLevelClient client, String indexName, String dataName, Consumer<Map<String, Object>> datasetTransform, | ||
CheckedBiFunction<XContent, InputStream, XContentParser, IOException> p) throws IOException { | ||
String name = "/data/" + indexName + ".mapping"; | ||
URL mapping = DataLoader.class.getResource(name); | ||
if (mapping == null) { | ||
throw new IllegalArgumentException("Cannot find resource " + name); | ||
} | ||
name = "/data/" + indexName + ".data"; | ||
name = "/data/" + (dataName != null ? dataName : indexName) + ".data"; | ||
URL data = DataLoader.class.getResource(name); | ||
if (data == null) { | ||
throw new IllegalArgumentException("Cannot find resource " + name); | ||
} | ||
createTestIndex(client, indexName, readMapping(mapping)); | ||
loadData(client, indexName, winFileTime, data, p); | ||
loadData(client, indexName, datasetTransform, data, p); | ||
} | ||
|
||
private static void createTestIndex(RestHighLevelClient client, String indexName, String mapping) throws IOException { | ||
|
@@ -147,8 +151,8 @@ private static CharSequence randomOf(String...values) { | |
} | ||
|
||
@SuppressWarnings("unchecked") | ||
private static void loadData(RestHighLevelClient client, String indexName, boolean winfileTime, URL resource, | ||
CheckedBiFunction<XContent, InputStream, XContentParser, IOException> p) | ||
private static void loadData(RestHighLevelClient client, String indexName, Consumer<Map<String, Object>> datasetTransform, | ||
URL resource, CheckedBiFunction<XContent, InputStream, XContentParser, IOException> p) | ||
throws IOException { | ||
BulkRequest bulk = new BulkRequest(); | ||
bulk.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); | ||
|
@@ -158,8 +162,8 @@ private static void loadData(RestHighLevelClient client, String indexName, boole | |
for (Object item : list) { | ||
assertThat(item, instanceOf(Map.class)); | ||
Map<String, Object> entry = (Map<String, Object>) item; | ||
if (winfileTime) { | ||
transformDataset(entry); | ||
if (datasetTransform != null) { | ||
datasetTransform.accept(entry); | ||
} | ||
bulk.add(new IndexRequest(indexName).source(entry, XContentType.JSON)); | ||
} | ||
|
@@ -175,14 +179,28 @@ private static void loadData(RestHighLevelClient client, String indexName, boole | |
} | ||
} | ||
|
||
private static void transformDataset(Map<String, Object> entry) { | ||
private static void timestampToUnixMillis(Map<String, Object> entry) { | ||
Object object = entry.get("timestamp"); | ||
assertThat(object, instanceOf(Long.class)); | ||
Long ts = (Long) object; | ||
// currently this is windows filetime | ||
entry.put("@timestamp", winFileTimeToUnix(ts)); | ||
} | ||
|
||
private static void timestampToUnixNanos(Map<String, Object> entry) { | ||
Object object = entry.get("timestamp"); | ||
assertThat(object, instanceOf(Long.class)); | ||
// interpret the value as nanos since the unix epoch | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The chosen Windows filetime timestamps (2017+) can coincidently also be readily used as nano-resolution unix timestamps (1973+). Which warrants conveniently using the same |
||
String timestamp = object.toString(); | ||
assertThat(timestamp.length(), greaterThan(12)); | ||
// avoid double approximations and BigDecimal ops | ||
String millis = timestamp.substring(0, 12); | ||
String milliFraction = timestamp.substring(12); | ||
// strip the fractions right away if not actually present | ||
entry.put("@timestamp", milliFraction.equals("000000") ? millis : millis + "." + milliFraction); | ||
entry.put("timestamp", ((long) object)/1_000_000L); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why this change? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. For There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I meant why do you need There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Just like for the existing millis tests, |
||
} | ||
|
||
public static long winFileTimeToUnix(final long filetime) { | ||
long ts = (filetime / FILETIME_ONE_MILLISECOND); | ||
return ts - FILETIME_EPOCH_DIFF; | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
👍 for the explanations