More details in tests

Author: kelvich

tests2/docker-compose.yml

@@ -11,12 +11,7 @@ services:
     environment:
       POSTGRES_USER: 'pg'
       POSTGRES_DB: 'regression'
-      MAJOR: 'off'
       NODE_ID: 1
-      # CONNSTRS: >-
-      #   dbname=regression user=pg host=node1,
-      #   dbname=regression user=pg host=toxi port=12000 arbiterport=12001,
-      #   dbname=regression user=pg host=toxi port=13000 arbiterport=13001
       CONNSTRS: >-
         dbname=regression user=pg host=node1,
         dbname=regression user=pg host=node2,
@@ -33,12 +28,7 @@ services:
     environment:
       POSTGRES_USER: 'pg'
       POSTGRES_DB: 'regression'
-      MAJOR: 'off'
       NODE_ID: 2
-      # CONNSTRS: >-
-      #   dbname=regression user=pg host=toxi port=21000 arbiterport=21001,
-      #   dbname=regression user=pg host=node2,
-      #   dbname=regression user=pg host=toxi port=23000 arbiterport=23001
       CONNSTRS: >-
         dbname=regression user=pg host=node1,
         dbname=regression user=pg host=node2,
@@ -55,12 +45,7 @@ services:
     environment:
       POSTGRES_USER: 'pg'
       POSTGRES_DB: 'regression'
-      MAJOR: 'off'
       NODE_ID: 3
-      # CONNSTRS: >-
-      #   dbname=regression user=pg host=toxi port=31000 arbiterport=31001,
-      #   dbname=regression user=pg host=toxi port=32000 arbiterport=32001,
-      #   dbname=regression user=pg host=node3
       CONNSTRS: >-
         dbname=regression user=pg host=node1,
         dbname=regression user=pg host=node2,

tests2/docker-entrypoint.sh

@@ -61,12 +61,9 @@ if [ "$1" = 'postgres' ]; then
 			log_line_prefix = '%m: '
 			# log_statement = all
 
-			multimaster.workers = 4
-			multimaster.max_workers = 16
 			multimaster.max_nodes = 3
-			multimaster.volkswagen_mode = 1
-			multimaster.queue_size=52857600
-			multimaster.ignore_tables_without_pk = 1
+			# multimaster.volkswagen_mode = 1
+			# multimaster.ignore_tables_without_pk = 1
 			multimaster.heartbeat_recv_timeout = 1100
 			multimaster.heartbeat_send_timeout = 250
 			multimaster.max_recovery_lag = 1000000000

tests2/lib/bank_client.py

@@ -207,7 +207,7 @@ def exec_tx(self, tx_block, aggname_prefix, conn_i):
                 if status != TRANSACTION_STATUS_IDLE:
                     yield from cur.execute('rollback')
 
-                yield from tx_block(conn, cur, agg)
+                yield from tx_block(conn, cur, agg, conn_i)
                 agg.finish_tx('commit')
 
             except psycopg2.Error as e:
@@ -231,7 +231,7 @@ def exec_tx(self, tx_block, aggname_prefix, conn_i):
         print("We've count to infinity!")
 
     @asyncio.coroutine
-    def transfer_tx(self, conn, cur, agg):
+    def transfer_tx(self, conn, cur, agg, conn_i):
         amount = 1
         # to avoid deadlocks:
         from_uid = random.randint(1, self.n_accounts - 2)
@@ -250,14 +250,14 @@ def transfer_tx(self, conn, cur, agg):
         yield from cur.execute('commit')
 
     @asyncio.coroutine
-    def total_tx(self, conn, cur, agg):
-        yield from cur.execute('select sum(amount) from bank_test')
+    def total_tx(self, conn, cur, agg, conn_i):
+        yield from cur.execute("select sum(amount), count(*), count(uid), current_setting('multimaster.node_id') from bank_test")
         total = yield from cur.fetchone()
         if total[0] != self.total:
             agg.isolation += 1
             self.total = total[0]
             print(self.oops)
-            print('Isolation error, total = ', total[0])
+            print(datetime.datetime.utcnow(), 'Isolation error, total = ', total, ', node ', conn_i+1)
             # yield from cur.execute('select * from mtm.get_nodes_state()')
             # nodes_state = yield from cur.fetchall()
             # for i, col in enumerate(self.nodes_state_fields):

tests2/test_recovery.py

@@ -169,6 +169,27 @@ def test_node_crash(self):
         self.assertCommits(aggs)
         self.assertIsolation(aggs)
 
+    def test_node_bicrash(self):
+        print('### test_node_bicrash ###')
+
+        aggs_failure, aggs = self.performFailure(CrashRecoverNode('node3'))
+
+        self.assertCommits(aggs_failure[:2])
+        self.assertNoCommits(aggs_failure[2:])
+        self.assertIsolation(aggs_failure)
+
+        self.assertCommits(aggs)
+        self.assertIsolation(aggs)
+
+        aggs_failure, aggs = self.performFailure(CrashRecoverNode('node3'))
+
+        self.assertCommits(aggs_failure[:2])
+        self.assertNoCommits(aggs_failure[2:])
+        self.assertIsolation(aggs_failure)
+
+        self.assertCommits(aggs)
+        self.assertIsolation(aggs)
+
 if __name__ == '__main__':
     unittest.main()
 

tests2/test_referee.py

@@ -61,6 +61,19 @@ def setUp(self):
     def tearDown(self):
         print('Finish test at ',datetime.datetime.utcnow())
 
+    def test_node_crash(self):
+        print('### test_node_crash ###')
+
+        aggs_failure, aggs = self.performFailure(CrashRecoverNode('node2'))
+
+        self.assertCommits(aggs_failure[:1])
+        self.assertNoCommits(aggs_failure[1:])
+        self.assertIsolation(aggs_failure)
+
+        self.assertCommits(aggs)
+        self.assertIsolation(aggs)
+
+
     def test_partition_referee(self):
         print('### test_partition_referee ###')