Feature/gtid gap log

src/Makefile

@@ -322,7 +322,7 @@ endif
 REDIS_SERVER_NAME=redis-server$(PROG_SUFFIX)
 REDIS_SENTINEL_NAME=redis-sentinel$(PROG_SUFFIX)
 REDIS_SWAP_OBJ=ctrip_swap.o ctrip_swap_adlist.o ctrip_lru_cache.o ctrip_swap_async.o ctrip_swap_batch.o ctrip_swap_cmd.o ctrip_swap_data.o ctrip_swap_debug.o ctrip_swap_evict.o ctrip_swap_exec.o ctrip_swap_expire.o ctrip_swap_hash.o ctrip_swap_set.o ctrip_swap_list.o ctrip_swap_iter.o ctrip_swap_zset.o ctrip_swap_meta.o ctrip_swap_object.o ctrip_swap_rdb.o ctrip_swap_repl.o ctrip_swap_rio.o ctrip_swap_rocks.o ctrip_swap_stat.o ctrip_swap_sync.o ctrip_swap_thread.o ctrip_swap_util.o ctrip_swap_lock.o ctrip_swap_string.o ctrip_swap_bitmap.o ctrip_swap_compact.o  ctrip_swap_slowlog.o ctrip_swap_blocked.o ctrip_cuckoo_hash.o ctrip_cuckoo_filter.o ctrip_swap_filter.o ctrip_absent_cache.o ctrip_swap_load.o ctrip_swap_dirty.o ctrip_swap_persist.o ctrip_roaring_bitmap.o ctrip_swap_rordb.o ctrip_wtdigest.o ctrip_swap_server.o
-REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o ctrip.o xredis_gtid.o xredis_gtid_aof.o xredis_gtid_repl.o xredis_gtid_rs.o xredis_gtid_rdb.o ctrip_heartbeat.o
+REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o ctrip.o xredis_gtid.o xredis_gtid_repl.o xredis_gtid_rs.o xredis_gtid_rdb.o xredis_gtid_gap_log.o xredis_gtid_adaptation_version.o ctrip_heartbeat.o xredis_gtid_cmdparse.o
 
 ifdef SWAP
 REDIS_SERVER_OBJ+= $(REDIS_SWAP_OBJ)

src/config.c

@@ -3190,6 +3190,7 @@ standardConfig configs[] = {
 
     /* ctrip configs */
     createBoolConfig("gtid-enabled", NULL, MODIFIABLE_CONFIG, server.gtid_enabled, 0, NULL, updateGtidEnabled),
+    createBoolConfig("gtid-gaplog-enabled", NULL, MODIFIABLE_CONFIG, server.gtid_gaplog_enabled, 1, NULL, NULL),
 
 #ifdef USE_OPENSSL
     createIntConfig("tls-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.tls_port, 0, INTEGER_CONFIG, NULL, updateTLSPort), /* TCP port. */

src/networking.c

@@ -501,20 +501,20 @@ void afterErrorReply(client *c, const char *s, size_t len) {
  * Unlike addReplyErrorSds and others alike which rely on addReplyErrorLength. */
 void addReplyErrorObject(client *c, robj *err) {
     addReply(c, err);
-    ctrip_afterErrorReply(c, err->ptr, sdslen(err->ptr)-2); /* Ignore trailing \r\n */
+    ctrip_afterErrorReply(c, err->ptr, sdslen(err->ptr)-2, -1); /* Ignore trailing \r\n */
 }
 
 /* See addReplyErrorLength for expectations from the input string. */
 void addReplyError(client *c, const char *err) {
     addReplyErrorLength(c,err,strlen(err));
-    ctrip_afterErrorReply(c,err,strlen(err));
+    ctrip_afterErrorReply(c,err,strlen(err), -1);
 }
 
 /* See addReplyErrorLength for expectations from the input string. */
 /* As a side effect the SDS string is freed. */
 void addReplyErrorSds(client *c, sds err) {
     addReplyErrorLength(c,err,sdslen(err));
-    ctrip_afterErrorReply(c,err,sdslen(err));
+    ctrip_afterErrorReply(c,err,sdslen(err), -1);
     sdsfree(err);
 }
 
@@ -531,7 +531,7 @@ void addReplyErrorFormat(client *c, const char *fmt, ...) {
      * invalid protocol is emitted. */
     s = sdsmapchars(s, "\r\n", "  ",  2);
     addReplyErrorLength(c,s,sdslen(s));
-    ctrip_afterErrorReply(c,s,sdslen(s));
+    ctrip_afterErrorReply(c,s,sdslen(s), -1);
     sdsfree(s);
 }
 
@@ -1479,7 +1479,7 @@ void freeClient(client *c) {
         //TODO what if master link reset but no master mode enabled?
         serverReplStreamSwitchIfNeeded(
                 server.gtid_enabled ? REPL_MODE_XSYNC:REPL_MODE_PSYNC,
-                RS_UPDATE_NOP,"master mode enabled(defer)");
+                RS_UPDATE_DOWN,"master mode enabled(defer)");
 
         if (!(c->flags & (CLIENT_PROTOCOL_ERROR|CLIENT_BLOCKED|CLIENT_SWAP_DISCARD_CACHED_MASTER))
                 && server.repl_mode->mode != REPL_MODE_XSYNC) {

src/redis_gtid.h

@@ -0,0 +1 @@
+../deps/xredis-gtid/xredis/redis_gtid_6x.h

src/replication.c

@@ -811,7 +811,7 @@ void syncCommand(client *c) {
      * So the slave knows the new replid and offset to try a PSYNC later
      * if the connection with the master is lost. */
     if (!strcasecmp(c->argv[0]->ptr,"psync") || !strcasecmp(c->argv[0]->ptr,"xsync")) {
-        if (ctrip_masterTryPartialResynchronization(c) == C_OK) {
+        if (ctrip_masterTryPartialResynchronization(c, PSYNC_OFFSET_UNSET) == C_OK) {
             server.stat_sync_partial_ok++;
             return; /* No full resync needed, return. */
         } else {
@@ -2808,6 +2808,11 @@ void replicationUnsetMaster(void) {
     if (server.master) freeClient(server.master);
     replicationDiscardCachedMaster();
     cancelReplicationHandshake(0);
+    /* Disconnecting all the slaves is required: we need to inform slaves
+     * of the replication ID change (see shiftReplicationId() call). However
+     * the slaves will be able to partially resync with us, so it will be
+     * a very fast reconnection. */
+    disconnectSlaves();
     /* When a slave is turned into a master, the current replication ID
      * (that was inherited from the master at synchronization time) is
      * used as secondary ID up to the current offset, and a new replication
@@ -2832,11 +2837,6 @@ void replicationUnsetMaster(void) {
                 server.gtid_enabled ? REPL_MODE_XSYNC:REPL_MODE_PSYNC,
                 RS_UPDATE_NOP,"master mode enabled");
     }
-    /* Disconnecting all the slaves is required: we need to inform slaves
-     * of the replication ID change (see shiftReplicationId() call). However
-     * the slaves will be able to partially resync with us, so it will be
-     * a very fast reconnection. */
-    disconnectSlaves();
     server.repl_state = REPL_STATE_NONE;
 
     /* We need to make sure the new master will start the replication stream

src/server.c

@@ -3521,6 +3521,7 @@ void initServer(void) {
     server.gtid_lost = gtidSetNew();
     xsyncUuidInterestedInit();
     gtidInitialInfoInit(server.gtid_initial);
+    server.gtid_gap_log = gtidGaplogNew();
     server.gtid_xsync_fullresync_indicator = 0;
     server.gtid_executed_cmd_count = 0;
     server.gtid_ignored_cmd_count = 0;

src/server.h

@@ -1741,6 +1741,9 @@ struct redisServer {
     long long gtid_ignored_cmd_count;
     long long gtid_executed_cmd_count;
     long long gtid_sync_stat[GTID_SYNC_TYPES];
+    int gtid_gaplog_enabled;
+    gtidGaplog* gtid_gap_log;
+
     /* importing mode */
     mstime_t importing_end_time;  /* in milliseconds */
     int importing_expire_enabled; 

src/xredis_commands.def

@@ -0,0 +1 @@
+../deps/xredis-gtid/xredis/xredis_commands.def

src/xredis_gtid_adaptation_version.c

@@ -0,0 +1 @@
+../deps/xredis-gtid/xredis/xredis_gtid_adaptation_version_6x.c

src/xredis_gtid_adaptation_version.h

@@ -0,0 +1 @@
+../deps/xredis-gtid/xredis/xredis_gtid_adaptation_version.h

src/xredis_gtid_cmdparse.c

@@ -0,0 +1 @@
+../deps/xredis-gtid/xredis/xredis_gtid_cmdparse.c

src/xredis_gtid_cmdparse.h

@@ -0,0 +1 @@
+../deps/xredis-gtid/xredis/xredis_gtid_cmdparse.h

src/xredis_gtid_gap_log.c

@@ -0,0 +1 @@
+../deps/xredis-gtid/xredis/xredis_gtid_gap_log.c

tests/support/util.tcl

@@ -13,8 +13,8 @@ proc randstring {min max {type binary}} {
     }
     while {$len} {
         set rr [expr {$minval+int(rand()*($maxval-$minval+1))}]
-        if {$type eq {alpha} && $rr eq 92} {
-            set rr 90; # avoid putting '\' char in the string, it can mess up TCL processing
+        if {$type eq {alpha} && ($rr eq 92 || $rr eq 91 || $rr eq 93)} {
+            set rr 94; # avoid '\', '[', ']' — '[' and ']' break TCL command substitution
         }
         append output [format "%c" $rr]
         incr len -1

tests/test_helper.tcl

@@ -18,6 +18,9 @@ set ::gtid_tests {
 	gtid/replication-psync
 	gtid/sync
 	gtid/xsync
+	gtid/gaplog
+    gtid/gaplog_commands
+    gtid/gaplog_write_commands
 }
 
 set ::all_tests {