Skip to content

Commit

Permalink
refact(rocksdb): clean & reformat some code (#2200)
Browse files Browse the repository at this point in the history 
* chore: merge master to clean-rocksdb for synchronization (#2383)

---------

Co-authored-by: V_Galaxy <dyc1904821183@gmail.com>
  • Loading branch information
@imbajin @VGalaxies
imbajin and VGalaxies authored Dec 11, 2023
1 parent bfe9fae commit bd5d68f
Show file tree
Hide file tree
Showing 21 changed files with 320 additions and 492 deletions.
14 changes: 7 additions & 7 deletions ...-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.NumericUtil;
import org.apache.hugegraph.util.StringEncoding;

import com.google.common.collect.ImmutableList;

public abstract class BackendTable<Session extends BackendSession, Entry> {
Expand Down Expand Up @@ -91,7 +92,8 @@ public void updateIfAbsent(Session session, Entry entry) {
}

/**
* Mapping query-type to table-type
* Mapping query-type to table-type
*
* @param query origin query
* @return corresponding table type
*/
Expand Down Expand Up @@ -231,12 +233,11 @@ protected long maxKey() {

public static class Range {

private byte[] startKey;
private byte[] endKey;
private final byte[] startKey;
private final byte[] endKey;

public Range(byte[] startKey, byte[] endKey) {
this.startKey = Arrays.equals(EMPTY, startKey) ?
START_BYTES : startKey;
this.startKey = Arrays.equals(EMPTY, startKey) ? START_BYTES : startKey;
this.endKey = Arrays.equals(EMPTY, endKey) ? END_BYTES : endKey;
}

Expand Down Expand Up @@ -361,8 +362,7 @@ public static byte[] increase(byte[] array) {
private static byte[] align(byte[] array, int length) {
int len = array.length;
E.checkArgument(len <= length,
"The length of array '%s' exceed " +
"align length '%s'", len, length);
"The length of array '%s' exceed align length '%s'", len, length);
byte[] target = new byte[length];
System.arraycopy(array, 0, target, length - len, len);
return target;
Expand Down
20 changes: 8 additions & 12 deletions ...graph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/OpenedRocksDB.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,16 @@
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.commons.io.FileUtils;
import org.apache.hugegraph.backend.BackendException;
import org.apache.hugegraph.backend.store.rocksdb.RocksDBIteratorPool.ReusedRocksIterator;
import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.Log;
import org.rocksdb.Checkpoint;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.RocksDB;
import org.rocksdb.SstFileManager;
import org.slf4j.Logger;

import org.apache.hugegraph.backend.BackendException;
import org.apache.hugegraph.backend.store.rocksdb.RocksDBIteratorPool.ReusedRocksIterator;
import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.Log;

public class OpenedRocksDB implements AutoCloseable {

private static final Logger LOG = Log.logger(OpenedRocksDB.class);
Expand Down Expand Up @@ -118,8 +117,7 @@ public void createCheckpoint(String targetPath) {
tempFile, snapshotFile));
}
} catch (Exception e) {
throw new BackendException("Failed to create checkpoint at path %s",
e, targetPath);
throw new BackendException("Failed to create checkpoint at path %s", e, targetPath);
}
}

Expand All @@ -137,8 +135,7 @@ public CFHandle(RocksDB rocksdb, ColumnFamilyHandle handle) {
}

public synchronized ColumnFamilyHandle get() {
E.checkState(this.handle.isOwningHandle(),
"It seems CF has been closed");
E.checkState(this.handle.isOwningHandle(), "It seems CF has been closed");
assert this.refs.get() >= 1;
return this.handle;
}
Expand All @@ -163,7 +160,7 @@ public void close() {

public synchronized ColumnFamilyHandle waitForDrop() {
assert this.refs.get() >= 1;
// When entering this method, the refs won't increase any more
// When entering this method, the refs won't increase anymore
final long timeout = TimeUnit.MINUTES.toMillis(30L);
final long unit = 100L;
for (long i = 1; this.refs.get() > 1; i++) {
Expand All @@ -173,8 +170,7 @@ public synchronized ColumnFamilyHandle waitForDrop() {
// 30s rest api timeout may cause InterruptedException
}
if (i * unit > timeout) {
throw new BackendException("Timeout after %sms to drop CF",
timeout);
throw new BackendException("Timeout after %sms to drop CF", timeout);
}
}
assert this.refs.get() == 1;
Expand Down
14 changes: 5 additions & 9 deletions ...aph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBIngester.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,14 @@
import java.util.ArrayList;
import java.util.List;

import org.apache.hugegraph.backend.BackendException;
import org.apache.hugegraph.util.Log;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.IngestExternalFileOptions;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
import org.slf4j.Logger;

import org.apache.hugegraph.backend.BackendException;
import org.apache.hugegraph.util.Log;

public class RocksDBIngester {

public static final String SST = ".sst";
Expand All @@ -52,8 +51,7 @@ public RocksDBIngester(RocksDB rocksdb) {
this.options.setMoveFiles(true);
}

public List<String> ingest(Path path, ColumnFamilyHandle cf)
throws RocksDBException {
public List<String> ingest(Path path, ColumnFamilyHandle cf) throws RocksDBException {
SuffixFileVisitor visitor = new SuffixFileVisitor(SST);
try {
Files.walkFileTree(path, visitor);
Expand All @@ -74,10 +72,8 @@ public List<String> ingest(Path path, ColumnFamilyHandle cf)
return ssts;
}

public void ingest(ColumnFamilyHandle cf, List<String> ssts)
throws RocksDBException {
LOG.info("Ingest sst files to CF '{}': {}",
RocksDBStdSessions.decode(cf.getName()), ssts);
public void ingest(ColumnFamilyHandle cf, List<String> ssts) throws RocksDBException {
LOG.info("Ingest sst files to CF '{}': {}", RocksDBStdSessions.decode(cf.getName()), ssts);
if (!ssts.isEmpty()) {
this.rocksdb.ingestExternalFile(cf, ssts, this.options);
}
Expand Down
20 changes: 9 additions & 11 deletions ...rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBIteratorPool.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,16 @@
import java.util.Queue;
import java.util.concurrent.ArrayBlockingQueue;

import org.apache.hugegraph.backend.BackendException;
import org.apache.hugegraph.config.CoreOptions;
import org.apache.hugegraph.util.Log;
import org.apache.hugegraph.util.StringEncoding;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
import org.rocksdb.RocksIterator;
import org.slf4j.Logger;

import org.apache.hugegraph.backend.BackendException;
import org.apache.hugegraph.config.CoreOptions;
import org.apache.hugegraph.util.Log;
import org.apache.hugegraph.util.StringEncoding;

public final class RocksDBIteratorPool implements AutoCloseable {

private static final Logger LOG = Log.logger(RocksDBIteratorPool.class);
Expand Down Expand Up @@ -63,9 +62,8 @@ public ReusedRocksIterator newIterator() {

@Override
public void close() {
LOG.debug("Close IteratorPool with pool size {} ({})",
this.pool.size(), this);
for (RocksIterator iter; (iter = this.pool.poll()) != null;) {
LOG.debug("Close IteratorPool with pool size {} ({})", this.pool.size(), this);
for (RocksIterator iter; (iter = this.pool.poll()) != null; ) {
this.closeIterator(iter);
}
assert this.pool.isEmpty();
Expand Down Expand Up @@ -149,13 +147,13 @@ private void closeIterator(RocksIterator iter) {

protected final class ReusedRocksIterator {

private static final boolean EREUSING_ENABLED = false;
private static final boolean REUSING_ENABLED = false;
private final RocksIterator iterator;
private boolean closed;

public ReusedRocksIterator() {
this.closed = false;
if (EREUSING_ENABLED) {
if (REUSING_ENABLED) {
this.iterator = allocIterator();
} else {
this.iterator = createIterator();
Expand All @@ -173,7 +171,7 @@ public void close() {
}
this.closed = true;

if (EREUSING_ENABLED) {
if (REUSING_ENABLED) {
releaseIterator(this.iterator);
} else {
closeIterator(this.iterator);
Expand Down
79 changes: 28 additions & 51 deletions ...raph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBMetrics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.hugegraph.util.Bytes;
import org.apache.hugegraph.util.InsertionOrderUtil;
import org.apache.hugegraph.util.UnitUtil;

import com.google.common.collect.ImmutableMap;

public class RocksDBMetrics implements BackendMetrics {
Expand All @@ -32,61 +33,37 @@ public class RocksDBMetrics implements BackendMetrics {
private static final String PREFIX = "rocksdb.";

// memory
private static final String BLOCK_CACHE = PREFIX +
"block-cache-usage";
private static final String BLOCK_CACHE_PINNED = PREFIX +
"block-cache-pinned-usage";
private static final String BLOCK_CACHE_CAPACITY = PREFIX +
"block-cache-capacity";
private static final String INDEX_FILTER = PREFIX +
"estimate-table-readers-mem";
private static final String ALL_MEM_TABLE = PREFIX +
"size-all-mem-tables";
private static final String CUR_MEM_TABLE = PREFIX +
"cur-size-all-mem-tables";
private static final String BLOCK_CACHE = PREFIX + "block-cache-usage";
private static final String BLOCK_CACHE_PINNED = PREFIX + "block-cache-pinned-usage";
private static final String BLOCK_CACHE_CAPACITY = PREFIX + "block-cache-capacity";
private static final String INDEX_FILTER = PREFIX + "estimate-table-readers-mem";
private static final String ALL_MEM_TABLE = PREFIX + "size-all-mem-tables";
private static final String CUR_MEM_TABLE = PREFIX + "cur-size-all-mem-tables";
// disk
private static final String DISK_USAGE = PREFIX +
"disk-usage";
private static final String LIVE_DATA_SIZE = PREFIX +
"estimate-live-data-size";
private static final String SST_FILE_SIZE = PREFIX +
"total-sst-files-size";
private static final String LIVE_SST_FILE_SIZE = PREFIX +
"live-sst-files-size";
private static final String DISK_USAGE = PREFIX + "disk-usage";
private static final String LIVE_DATA_SIZE = PREFIX + "estimate-live-data-size";
private static final String SST_FILE_SIZE = PREFIX + "total-sst-files-size";
private static final String LIVE_SST_FILE_SIZE = PREFIX + "live-sst-files-size";
private static final String PENDING_COMPACTION_BYTES = PREFIX +
"estimate-pending-compaction-bytes";
"estimate-pending-compaction-bytes";

// count/number
private static final String NUM_KEYS = PREFIX +
"estimate-num-keys";
private static final String NUM_KEYS_MEM_TABLE = PREFIX +
"num-entries-active-mem-table";
private static final String NUM_KEYS_IMM_MEM_TABLE = PREFIX +
"num-entries-imm-mem-tables";
private static final String NUM_DELETES_MEM_TABLE = PREFIX +
"num-deletes-active-mem-table";
private static final String NUM_DELETES_IMM_MEM_TABLE = PREFIX +
"num-deletes-imm-mem-tables";

private static final String RUNNING_FLUSHS = PREFIX +
"num-running-flushes";
private static final String MEM_TABLE_FLUSH_PENDINF = PREFIX +
"mem-table-flush-pending";
private static final String RUNNING_COMPACTIONS = PREFIX +
"num-running-compactions";
private static final String COMPACTION_PENDINF = PREFIX +
"compaction-pending";

private static final String NUM_IMM_MEM_TABLE = PREFIX +
"num-immutable-mem-table";
private static final String NUM_SNAPSHOTS = PREFIX +
"num-snapshots";
private static final String OLDEST_SNAPSHOT_TIME = PREFIX +
"oldest-snapshot-time";
private static final String NUM_LIVE_VERSIONS = PREFIX +
"num-live-versions";
private static final String SUPER_VERSION = PREFIX +
"current-super-version-number";
private static final String NUM_KEYS = PREFIX + "estimate-num-keys";
private static final String NUM_KEYS_MEM_TABLE = PREFIX + "num-entries-active-mem-table";
private static final String NUM_KEYS_IMM_MEM_TABLE = PREFIX + "num-entries-imm-mem-tables";
private static final String NUM_DELETES_MEM_TABLE = PREFIX + "num-deletes-active-mem-table";
private static final String NUM_DELETES_IMM_MEM_TABLE = PREFIX + "num-deletes-imm-mem-tables";

private static final String RUNNING_FLUSHS = PREFIX + "num-running-flushes";
private static final String MEM_TABLE_FLUSH_PENDINF = PREFIX + "mem-table-flush-pending";
private static final String RUNNING_COMPACTIONS = PREFIX + "num-running-compactions";
private static final String COMPACTION_PENDINF = PREFIX + "compaction-pending";

private static final String NUM_IMM_MEM_TABLE = PREFIX + "num-immutable-mem-table";
private static final String NUM_SNAPSHOTS = PREFIX + "num-snapshots";
private static final String OLDEST_SNAPSHOT_TIME = PREFIX + "oldest-snapshot-time";
private static final String NUM_LIVE_VERSIONS = PREFIX + "num-live-versions";
private static final String SUPER_VERSION = PREFIX + "current-super-version-number";

public static final String KEY_DISK_USAGE = DISK_USAGE;
public static final String KEY_NUM_KEYS = NUM_KEYS;
Expand Down
11 changes: 6 additions & 5 deletions ...raph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,17 @@
import static org.apache.hugegraph.config.OptionChecker.rangeDouble;
import static org.apache.hugegraph.config.OptionChecker.rangeInt;

import org.rocksdb.CompactionStyle;
import org.rocksdb.CompressionType;
import org.rocksdb.DataBlockIndexType;
import org.rocksdb.IndexType;

import org.apache.hugegraph.config.ConfigConvOption;
import org.apache.hugegraph.config.ConfigListConvOption;
import org.apache.hugegraph.config.ConfigListOption;
import org.apache.hugegraph.config.ConfigOption;
import org.apache.hugegraph.config.OptionHolder;
import org.apache.hugegraph.util.Bytes;
import org.rocksdb.CompactionStyle;
import org.rocksdb.CompressionType;
import org.rocksdb.DataBlockIndexType;
import org.rocksdb.IndexType;

import com.google.common.collect.ImmutableList;

public class RocksDBOptions extends OptionHolder {
Expand All @@ -52,6 +52,7 @@ public static synchronized RocksDBOptions instance() {
return instance;
}

// TODO: the entire align style is wrong, change it to 4 space later
public static final ConfigOption<String> DATA_PATH =
new ConfigOption<>(
"rocksdb.data_path",
Expand Down
25 changes: 8 additions & 17 deletions ...aph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBSessions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,11 @@
import java.util.Set;

import org.apache.commons.lang3.tuple.Pair;
import org.rocksdb.RocksDBException;

import org.apache.hugegraph.backend.store.BackendEntry.BackendColumnIterator;
import org.apache.hugegraph.backend.store.BackendSession.AbstractBackendSession;
import org.apache.hugegraph.backend.store.BackendSessionPool;
import org.apache.hugegraph.config.HugeConfig;
import org.rocksdb.RocksDBException;

public abstract class RocksDBSessions extends BackendSessionPool {

Expand All @@ -46,17 +45,15 @@ public RocksDBSessions(HugeConfig config, String database, String store) {

public abstract void compactRange();

public abstract RocksDBSessions copy(HugeConfig config,
String database, String store);
public abstract RocksDBSessions copy(HugeConfig config, String database, String store);

public abstract void createSnapshot(String snapshotPath);

public abstract void resumeSnapshot(String snapshotPath);

public abstract String buildSnapshotPath(String snapshotPrefix);

public abstract String hardLinkSnapshot(String snapshotPath)
throws RocksDBException;
public abstract String hardLinkSnapshot(String snapshotPath) throws RocksDBException;

public abstract void reloadRocksDB() throws RocksDBException;

Expand Down Expand Up @@ -105,22 +102,16 @@ public abstract void deleteRange(String table,

public abstract byte[] get(String table, byte[] key);

public abstract BackendColumnIterator get(String table,
List<byte[]> keys);
public abstract BackendColumnIterator get(String table, List<byte[]> keys);

public abstract BackendColumnIterator scan(String table);

public abstract BackendColumnIterator scan(String table,
byte[] prefix);
public abstract BackendColumnIterator scan(String table, byte[] prefix);

public abstract BackendColumnIterator scan(String table,
byte[] keyFrom,
byte[] keyTo,
int scanType);
public abstract BackendColumnIterator scan(String table, byte[] keyFrom,
byte[] keyTo, int scanType);

public BackendColumnIterator scan(String table,
byte[] keyFrom,
byte[] keyTo) {
public BackendColumnIterator scan(String table, byte[] keyFrom, byte[] keyTo) {
return this.scan(table, keyFrom, keyTo, SCAN_LT_END);
}

Expand Down
Loading

0 comments on commit bd5d68f

Please sign in to comment.