Showing content from https://gitlab.com/gitlab-org/gitlab-qa/-/merge_requests/90.patch below:
From d6ca0477e3657b368c7c4448f4f69f56792aa449 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 9 Nov 2017 11:05:25 +0100 Subject: [PATCH 01/19] Add initial geo integration test entrypoint --- lib/gitlab/qa/scenario/test/integration/geo.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 lib/gitlab/qa/scenario/test/integration/geo.rb diff --git a/lib/gitlab/qa/scenario/test/integration/geo.rb b/lib/gitlab/qa/scenario/test/integration/geo.rb new file mode 100644 index 000000000..89d4038ce --- /dev/null +++ b/lib/gitlab/qa/scenario/test/integration/geo.rb @@ -0,0 +1,18 @@ +module Gitlab + module QA + module Scenario + module Test + module Integration + class Geo < Scenario::Template + # rubocop:disable Metrics/MethodLength + def perform(release) + Component::Gitlab.perform do |gitlab| + gitlab.release = release + end + end + end + end + end + end + end +end -- GitLab From 16a4e05dc8ca5344caf85f0eb436084a9f2c861e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 9 Nov 2017 11:05:41 +0100 Subject: [PATCH 02/19] Make it easier to add multiple omnibus configs --- lib/gitlab/qa/component/gitlab.rb | 13 ++++++++++--- .../qa/scenario/test/integration/mattermost.rb | 5 +++-- spec/component/gitlab_spec.rb | 10 +++++----- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/lib/gitlab/qa/component/gitlab.rb b/lib/gitlab/qa/component/gitlab.rb index c35ff8ea3..04d87767a 100644 --- a/lib/gitlab/qa/component/gitlab.rb +++ b/lib/gitlab/qa/component/gitlab.rb @@ -14,7 +14,7 @@ module Gitlab # rubocop:disable Style/Semicolon attr_reader :release, :docker - attr_accessor :volumes, :network, :environment, :network_aliases + attr_accessor :volumes, :network, :environment def_delegators :release, :tag, :image, :edition @@ -23,12 +23,19 @@ module Gitlab @environment = {} @volumes = {} @network_aliases = [] + @omnibus_config = [] self.release = 'CE' end - def omnibus_config=(config) - @environment['GITLAB_OMNIBUS_CONFIG'] = config + def add_omnibus_config(config) + @omnibus_config.push(config).tap do |lines| + @environment['GITLAB_OMNIBUS_CONFIG'] = lines.join(';') + end + end + + def add_network_alias(name) + @network_aliases.push(name) end def release=(release) diff --git a/lib/gitlab/qa/scenario/test/integration/mattermost.rb b/lib/gitlab/qa/scenario/test/integration/mattermost.rb index f46d6d3bf..e5dbfef2c 100644 --- a/lib/gitlab/qa/scenario/test/integration/mattermost.rb +++ b/lib/gitlab/qa/scenario/test/integration/mattermost.rb @@ -11,9 +11,10 @@ module Gitlab mattermost_hostname = "mattermost.#{gitlab.network}" mattermost_external_url = "http://#{mattermost_hostname}" - gitlab.omnibus_config = - "mattermost_external_url '#{mattermost_external_url}'" + gitlab.network_aliases = [mattermost_hostname] + gitlab.add_omnibus_config( + "mattermost_external_url '#{mattermost_external_url}'") gitlab.instance do Component::Specs.perform do |specs| diff --git a/spec/component/gitlab_spec.rb b/spec/component/gitlab_spec.rb index 8a94d57f9..2abd7b870 100644 --- a/spec/component/gitlab_spec.rb +++ b/spec/component/gitlab_spec.rb @@ -2,13 +2,13 @@ describe Gitlab::QA::Component::Gitlab do let(:full_ce_address) { 'registry.gitlab.com/foo/gitlab/gitlab-ce' } let(:full_ce_address_with_complex_tag) { "#{full_ce_address}:omnibus-7263a2" } - describe '#omnibus_config=' do + describe '#add_omnibus_config' do context 'when set' do before do - subject.omnibus_config = '# Be configured' + subject.add_omnibus_config('# Be configured') end - it 'updates #environment' do + it 'updates environment variables' do expect(subject.environment['GITLAB_OMNIBUS_CONFIG']) .to eq('# Be configured') end @@ -163,9 +163,9 @@ describe Gitlab::QA::Component::Gitlab do end end - context 'with network_aliases' do + context 'with network_alias' do before do - subject.network_aliases = ['lolcathost'] + subject.add_network_alias('lolcathost') end it 'adds --network-alias switches to the command' do -- GitLab From 8d6364e38addafb772dd1cfc529d2471641ec022 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 9 Nov 2017 11:35:41 +0100 Subject: [PATCH 03/19] Configure primary and secondary nodes --- .../qa/scenario/test/integration/geo.rb | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/qa/scenario/test/integration/geo.rb b/lib/gitlab/qa/scenario/test/integration/geo.rb index 89d4038ce..3e98a1974 100644 --- a/lib/gitlab/qa/scenario/test/integration/geo.rb +++ b/lib/gitlab/qa/scenario/test/integration/geo.rb @@ -6,9 +6,43 @@ module Gitlab class Geo < Scenario::Template # rubocop:disable Metrics/MethodLength def perform(release) - Component::Gitlab.perform do |gitlab| - gitlab.release = release + release = Release.new(release) + + unless release.edition == :ee + raise ArgumentError, 'Geo is EE only!' end + + Component::Gitlab.perform do |primary| + primary.release = release + configure_primary(primary) + + primary.instance do + Component::Gitlab.perform do |secondary| + secondary.release = release + configure_secondary(secondary) + + secondary.instance do + # shellout to instance specs + end + end + end + end + end + + private + + def configure_primary(node) + node.add_omnibus_config("geo_primary_role['enable'] = true") + node.add_omnibus_config("postgresql['listen_address'] = '0.0.0.0'") + node.add_omnibus_config("postgresql['trust_auth_cidr_addresses'] = ['0.0.0.0/0','0.0.0.0/0']") + node.add_omnibus_config("postgresql['md5_auth_cidr_addresses'] = ['0.0.0.0/0']") + node.add_omnibus_config("postgresql['max_replication_slots'] = 1") + node.add_omnibus_config("gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc'") + end + + def configure_secondary(node) + node.add_omnibus_config("geo_secondary_role['enable'] = true") + node.add_omnibus_config("gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc'") end end end -- GitLab From 9b1f663da50edfa5bab5a2028a680037e342140c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 9 Nov 2017 11:43:12 +0100 Subject: [PATCH 04/19] DRY Geo test scenario --- lib/gitlab/qa/component/gitlab.rb | 7 ++--- .../qa/scenario/test/integration/geo.rb | 30 ++++++++----------- .../scenario/test/integration/mattermost.rb | 6 ++-- spec/component/gitlab_spec.rb | 4 +-- 4 files changed, 19 insertions(+), 28 deletions(-) diff --git a/lib/gitlab/qa/component/gitlab.rb b/lib/gitlab/qa/component/gitlab.rb index 04d87767a..e8a86e1b4 100644 --- a/lib/gitlab/qa/component/gitlab.rb +++ b/lib/gitlab/qa/component/gitlab.rb @@ -23,15 +23,12 @@ module Gitlab @environment = {} @volumes = {} @network_aliases = [] - @omnibus_config = [] self.release = 'CE' end - def add_omnibus_config(config) - @omnibus_config.push(config).tap do |lines| - @environment['GITLAB_OMNIBUS_CONFIG'] = lines.join(';') - end + def omnibus_config=(config) + @environment['GITLAB_OMNIBUS_CONFIG'] = config end def add_network_alias(name) diff --git a/lib/gitlab/qa/scenario/test/integration/geo.rb b/lib/gitlab/qa/scenario/test/integration/geo.rb index 3e98a1974..bafc3cc4a 100644 --- a/lib/gitlab/qa/scenario/test/integration/geo.rb +++ b/lib/gitlab/qa/scenario/test/integration/geo.rb @@ -14,12 +14,22 @@ module Gitlab Component::Gitlab.perform do |primary| primary.release = release - configure_primary(primary) + primary.omnibus_config = <<~OMNIBUS + geo_primary_role['enable'] = true; + postgresql['listen_address'] = '0.0.0.0'; + postgresql['trust_auth_cidr_addresses'] = ['0.0.0.0/0','0.0.0.0/0']; + postgresql['md5_auth_cidr_addresses'] = ['0.0.0.0/0']; + postgresql['max_replication_slots'] = 1; + gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc'; + OMNIBUS primary.instance do Component::Gitlab.perform do |secondary| secondary.release = release - configure_secondary(secondary) + secondary.omnibus_config = <<~OMNIBUS + geo_secondary_role['enable'] = true"; + gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc'; + OMNIBUS secondary.instance do # shellout to instance specs @@ -28,22 +38,6 @@ module Gitlab end end end - - private - - def configure_primary(node) - node.add_omnibus_config("geo_primary_role['enable'] = true") - node.add_omnibus_config("postgresql['listen_address'] = '0.0.0.0'") - node.add_omnibus_config("postgresql['trust_auth_cidr_addresses'] = ['0.0.0.0/0','0.0.0.0/0']") - node.add_omnibus_config("postgresql['md5_auth_cidr_addresses'] = ['0.0.0.0/0']") - node.add_omnibus_config("postgresql['max_replication_slots'] = 1") - node.add_omnibus_config("gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc'") - end - - def configure_secondary(node) - node.add_omnibus_config("geo_secondary_role['enable'] = true") - node.add_omnibus_config("gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc'") - end end end end diff --git a/lib/gitlab/qa/scenario/test/integration/mattermost.rb b/lib/gitlab/qa/scenario/test/integration/mattermost.rb index e5dbfef2c..022b1bb2d 100644 --- a/lib/gitlab/qa/scenario/test/integration/mattermost.rb +++ b/lib/gitlab/qa/scenario/test/integration/mattermost.rb @@ -11,10 +11,10 @@ module Gitlab mattermost_hostname = "mattermost.#{gitlab.network}" mattermost_external_url = "http://#{mattermost_hostname}" - gitlab.network_aliases = [mattermost_hostname] - gitlab.add_omnibus_config( - "mattermost_external_url '#{mattermost_external_url}'") + gitlab.omnibus_config = <<~OMNIBUS + mattermost_external_url '#{mattermost_external_url}' + OMNIBUS gitlab.instance do Component::Specs.perform do |specs| diff --git a/spec/component/gitlab_spec.rb b/spec/component/gitlab_spec.rb index 2abd7b870..63fcc0ebd 100644 --- a/spec/component/gitlab_spec.rb +++ b/spec/component/gitlab_spec.rb @@ -2,10 +2,10 @@ describe Gitlab::QA::Component::Gitlab do let(:full_ce_address) { 'registry.gitlab.com/foo/gitlab/gitlab-ce' } let(:full_ce_address_with_complex_tag) { "#{full_ce_address}:omnibus-7263a2" } - describe '#add_omnibus_config' do + describe '#omnibus_config=' do context 'when set' do before do - subject.add_omnibus_config('# Be configured') + subject.omnibus_config = '# Be configured' end it 'updates environment variables' do -- GitLab From 6bc71e727a8f71176191533e41c4df0a92a02fed Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 9 Nov 2017 11:45:13 +0100 Subject: [PATCH 05/19] Add Geo integration test entrypoint to the pipeline --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index deaf86c65..d267c9e5a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -95,6 +95,11 @@ ee:mattermost: script: bin/qa Test::Integration::Mattermost EE <<: *test +ee:geo: + script: bin/qa Test::Integration:Geo EE + allow_failure: true + <<: *test + notify:slack: before_script: - apk update && apk add git curl bash -- GitLab From 954f43eb6b2b329c2fca3fb1de9169fe1803a5db Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 9 Nov 2017 12:02:36 +0100 Subject: [PATCH 06/19] Update GitLab QA dependencies --- lib/gitlab/qa/component/gitlab.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/qa/component/gitlab.rb b/lib/gitlab/qa/component/gitlab.rb index e8a86e1b4..a97e62676 100644 --- a/lib/gitlab/qa/component/gitlab.rb +++ b/lib/gitlab/qa/component/gitlab.rb @@ -7,7 +7,7 @@ require 'shellwords' module Gitlab module QA module Component - class Gitlab + class Gitlab # rubocop:disable Metrics/ClassLength extend Forwardable include Scenario::Actable -- GitLab From 5904aabafd95c6cb7cc1e90bd23f4e8766f6c787 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 9 Nov 2017 12:50:16 +0100 Subject: [PATCH 07/19] Autoload Geo integration test scenario --- lib/gitlab/qa.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/gitlab/qa.rb b/lib/gitlab/qa.rb index baab2b392..6be181a3f 100644 --- a/lib/gitlab/qa.rb +++ b/lib/gitlab/qa.rb @@ -25,6 +25,7 @@ module Gitlab module Integration autoload :Mattermost, 'qa/scenario/test/integration/mattermost' + autoload :Geo, 'qa/scenario/test/integration/geo' end module Sanity -- GitLab From 4ee01bf057934a946c28933e3c692ea80c6754b5 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 9 Nov 2017 13:37:59 +0100 Subject: [PATCH 08/19] Fix network alias in mattermost test scenario --- lib/gitlab/qa/scenario/test/integration/mattermost.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/qa/scenario/test/integration/mattermost.rb b/lib/gitlab/qa/scenario/test/integration/mattermost.rb index 022b1bb2d..b9685c624 100644 --- a/lib/gitlab/qa/scenario/test/integration/mattermost.rb +++ b/lib/gitlab/qa/scenario/test/integration/mattermost.rb @@ -11,7 +11,8 @@ module Gitlab mattermost_hostname = "mattermost.#{gitlab.network}" mattermost_external_url = "http://#{mattermost_hostname}" - gitlab.network_aliases = [mattermost_hostname] + + gitlab.add_network_alias(mattermost_hostname) gitlab.omnibus_config = <<~OMNIBUS mattermost_external_url '#{mattermost_external_url}' OMNIBUS -- GitLab From da3b8cfec85bad6eba478a42f2520e7e55a0bfa9 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 9 Nov 2017 13:51:48 +0100 Subject: [PATCH 09/19] Fix constant name in .gitlab-ci.yml for Geo tests --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d267c9e5a..b8a1dbc23 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -96,7 +96,7 @@ ee:mattermost: <<: *test ee:geo: - script: bin/qa Test::Integration:Geo EE + script: bin/qa Test::Integration::Geo EE allow_failure: true <<: *test -- GitLab From 54467b5951cf10cc197e574fe2756e73361759fc Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 9 Nov 2017 14:11:19 +0100 Subject: [PATCH 10/19] Connect Geo nodes into single `geo` docker network --- lib/gitlab/qa/scenario/test/integration/geo.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/gitlab/qa/scenario/test/integration/geo.rb b/lib/gitlab/qa/scenario/test/integration/geo.rb index bafc3cc4a..38343a1b7 100644 --- a/lib/gitlab/qa/scenario/test/integration/geo.rb +++ b/lib/gitlab/qa/scenario/test/integration/geo.rb @@ -14,6 +14,7 @@ module Gitlab Component::Gitlab.perform do |primary| primary.release = release + primary.network = 'geo' primary.omnibus_config = <<~OMNIBUS geo_primary_role['enable'] = true; postgresql['listen_address'] = '0.0.0.0'; @@ -26,6 +27,7 @@ module Gitlab primary.instance do Component::Gitlab.perform do |secondary| secondary.release = release + secondary.network = 'geo' secondary.omnibus_config = <<~OMNIBUS geo_secondary_role['enable'] = true"; gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc'; -- GitLab From 328090eb4b1cd6a7bf4e081a51dea7425bbb090e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Sat, 11 Nov 2017 13:59:54 +0100 Subject: [PATCH 11/19] Fix Rubocop overrides --- lib/gitlab/qa/component/gitlab.rb | 2 +- lib/gitlab/qa/scenario/test/integration/geo.rb | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/gitlab/qa/component/gitlab.rb b/lib/gitlab/qa/component/gitlab.rb index a97e62676..e8a86e1b4 100644 --- a/lib/gitlab/qa/component/gitlab.rb +++ b/lib/gitlab/qa/component/gitlab.rb @@ -7,7 +7,7 @@ require 'shellwords' module Gitlab module QA module Component - class Gitlab # rubocop:disable Metrics/ClassLength + class Gitlab extend Forwardable include Scenario::Actable diff --git a/lib/gitlab/qa/scenario/test/integration/geo.rb b/lib/gitlab/qa/scenario/test/integration/geo.rb index 38343a1b7..e087221c9 100644 --- a/lib/gitlab/qa/scenario/test/integration/geo.rb +++ b/lib/gitlab/qa/scenario/test/integration/geo.rb @@ -4,8 +4,7 @@ module Gitlab module Test module Integration class Geo < Scenario::Template - # rubocop:disable Metrics/MethodLength - def perform(release) + def perform(release) # rubocop:disable Metrics/MethodLength release = Release.new(release) unless release.edition == :ee -- GitLab From b6d2fef6f51892d256e204edcba9da943cee7bd6 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Sat, 11 Nov 2017 14:13:01 +0100 Subject: [PATCH 12/19] Fix syntax error in Geo integrations test scenario --- lib/gitlab/qa/scenario/test/integration/geo.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/qa/scenario/test/integration/geo.rb b/lib/gitlab/qa/scenario/test/integration/geo.rb index e087221c9..bb2b40963 100644 --- a/lib/gitlab/qa/scenario/test/integration/geo.rb +++ b/lib/gitlab/qa/scenario/test/integration/geo.rb @@ -28,7 +28,7 @@ module Gitlab secondary.release = release secondary.network = 'geo' secondary.omnibus_config = <<~OMNIBUS - geo_secondary_role['enable'] = true"; + geo_secondary_role['enable'] = true; gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc'; OMNIBUS -- GitLab From dff604bc51a4ea1089469c14b70ac33e20241fb5 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 13 Nov 2017 09:12:50 +0100 Subject: [PATCH 13/19] Make docker commands with envs more readable --- lib/gitlab/qa/component/gitlab.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/gitlab/qa/component/gitlab.rb b/lib/gitlab/qa/component/gitlab.rb index e8a86e1b4..df8430494 100644 --- a/lib/gitlab/qa/component/gitlab.rb +++ b/lib/gitlab/qa/component/gitlab.rb @@ -2,7 +2,6 @@ require 'securerandom' require 'net/http' require 'uri' require 'forwardable' -require 'shellwords' module Gitlab module QA @@ -28,7 +27,7 @@ module Gitlab end def omnibus_config=(config) - @environment['GITLAB_OMNIBUS_CONFIG'] = config + @environment['GITLAB_OMNIBUS_CONFIG'] = config.gsub("\n", ' ') end def add_network_alias(name) @@ -83,8 +82,7 @@ module Gitlab end @environment.to_h.each do |key, value| - escaped_value = Shellwords.escape(value) - command << "--env #{key}=#{escaped_value}" + command << %Q[--env #{key}="#{value}"] end @network_aliases.to_a.each do |network_alias| -- GitLab From 72df55f127a17e07f0025f5c9af765f65c2956bc Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 13 Nov 2017 09:22:44 +0100 Subject: [PATCH 14/19] Fix rubocop offenses in GitLab component code --- lib/gitlab/qa/component/gitlab.rb | 4 ++-- spec/component/gitlab_spec.rb | 24 +++++++----------------- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/lib/gitlab/qa/component/gitlab.rb b/lib/gitlab/qa/component/gitlab.rb index df8430494..ab24e94d5 100644 --- a/lib/gitlab/qa/component/gitlab.rb +++ b/lib/gitlab/qa/component/gitlab.rb @@ -27,7 +27,7 @@ module Gitlab end def omnibus_config=(config) - @environment['GITLAB_OMNIBUS_CONFIG'] = config.gsub("\n", ' ') + @environment['GITLAB_OMNIBUS_CONFIG'] = config.tr("\n", ' ') end def add_network_alias(name) @@ -82,7 +82,7 @@ module Gitlab end @environment.to_h.each do |key, value| - command << %Q[--env #{key}="#{value}"] + command << %(--env #{key}="#{value}") end @network_aliases.to_a.each do |network_alias| diff --git a/spec/component/gitlab_spec.rb b/spec/component/gitlab_spec.rb index 63fcc0ebd..cba72ee64 100644 --- a/spec/component/gitlab_spec.rb +++ b/spec/component/gitlab_spec.rb @@ -135,31 +135,20 @@ describe Gitlab::QA::Component::Gitlab do it 'adds --volume switches to the command' do subject.start + expect(args).to include('--volume /from:/to:Z') end end context 'with environment' do - context 'plain values' do - before do - subject.environment = { 'TEST' => 'value' } - end - - it 'adds --env switches to the command' do - subject.start - expect(args).to include('--env TEST=value') - end + before do + subject.environment = { 'TEST' => 'a value with spaces' } end - context 'values with spaces' do - before do - subject.environment = { 'TEST' => 'a value with spaces' } - end + it 'adds quotes around env' do + subject.start - it 'adds --env shell escaped values' do - subject.start - expect(args).to include('--env TEST=a\ value\ with\ spaces') - end + expect(args).to include('--env TEST="a value with spaces"') end end @@ -170,6 +159,7 @@ describe Gitlab::QA::Component::Gitlab do it 'adds --network-alias switches to the command' do subject.start + expect(args).to include('--network-alias lolcathost') end end -- GitLab From d81a9e114a56d791d0cefb9a2168ff78801561b4 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 13 Nov 2017 10:06:06 +0100 Subject: [PATCH 15/19] WIP Geo secondary start-up debuggin mode --- .gitlab-ci.yml | 132 +++++++++--------- .../qa/scenario/test/integration/geo.rb | 57 +++++--- 2 files changed, 101 insertions(+), 88 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b8a1dbc23..854775c98 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,41 +16,41 @@ variables: before_script: - bundle install -check:rubocop: - stage: check - image: $TEST_IMAGE - script: bundle exec rubocop - except: - - triggers - tags: - - docker - -check:rspec: - stage: check - image: $TEST_IMAGE - script: bundle exec rspec - except: - - triggers - tags: - - docker - -.sanity: &sanity - stage: sanity - image: $TEST_IMAGE - except: - - triggers - tags: - - docker - -ce:version: - script: bin/qa Test::Sanity::Version CE - allow_failure: true - <<: *sanity - -ee:version: - script: bin/qa Test::Sanity::Version EE - allow_failure: true - <<: *sanity +# check:rubocop: +# stage: check +# image: $TEST_IMAGE +# script: bundle exec rubocop +# except: +# - triggers +# tags: +# - docker +# +# check:rspec: +# stage: check +# image: $TEST_IMAGE +# script: bundle exec rspec +# except: +# - triggers +# tags: +# - docker +# +# .sanity: &sanity +# stage: sanity +# image: $TEST_IMAGE +# except: +# - triggers +# tags: +# - docker +# +# ce:version: +# script: bin/qa Test::Sanity::Version CE +# allow_failure: true +# <<: *sanity +# +# ee:version: +# script: bin/qa Test::Sanity::Version EE +# allow_failure: true +# <<: *sanity .test: &test stage: test @@ -63,37 +63,37 @@ ee:version: paths: - screenshots/*.png -ce:instance: - script: bin/qa Test::Instance::Image CE - <<: *test - -ee:instance: - script: bin/qa Test::Instance::Image EE - <<: *test - -ce:image: - script: bin/qa Test::Omnibus::Image CE - <<: *test - -ee:image: - script: bin/qa Test::Omnibus::Image EE - <<: *test - -ce:upgrade: - script: bin/qa Test::Omnibus::Upgrade CE - <<: *test - -ee:upgrade: - script: bin/qa Test::Omnibus::Upgrade EE - <<: *test - -ce:mattermost: - script: bin/qa Test::Integration::Mattermost CE - <<: *test - -ee:mattermost: - script: bin/qa Test::Integration::Mattermost EE - <<: *test +# ce:instance: +# script: bin/qa Test::Instance::Image CE +# <<: *test +# +# ee:instance: +# script: bin/qa Test::Instance::Image EE +# <<: *test +# +# ce:image: +# script: bin/qa Test::Omnibus::Image CE +# <<: *test +# +# ee:image: +# script: bin/qa Test::Omnibus::Image EE +# <<: *test +# +# ce:upgrade: +# script: bin/qa Test::Omnibus::Upgrade CE +# <<: *test +# +# ee:upgrade: +# script: bin/qa Test::Omnibus::Upgrade EE +# <<: *test +# +# ce:mattermost: +# script: bin/qa Test::Integration::Mattermost CE +# <<: *test +# +# ee:mattermost: +# script: bin/qa Test::Integration::Mattermost EE +# <<: *test ee:geo: script: bin/qa Test::Integration::Geo EE diff --git a/lib/gitlab/qa/scenario/test/integration/geo.rb b/lib/gitlab/qa/scenario/test/integration/geo.rb index bb2b40963..3f61ab1c0 100644 --- a/lib/gitlab/qa/scenario/test/integration/geo.rb +++ b/lib/gitlab/qa/scenario/test/integration/geo.rb @@ -11,33 +11,46 @@ module Gitlab raise ArgumentError, 'Geo is EE only!' end - Component::Gitlab.perform do |primary| - primary.release = release - primary.network = 'geo' - primary.omnibus_config = <<~OMNIBUS - geo_primary_role['enable'] = true; - postgresql['listen_address'] = '0.0.0.0'; - postgresql['trust_auth_cidr_addresses'] = ['0.0.0.0/0','0.0.0.0/0']; - postgresql['md5_auth_cidr_addresses'] = ['0.0.0.0/0']; - postgresql['max_replication_slots'] = 1; + Component::Gitlab.perform do |secondary| + secondary.release = release + secondary.network = 'geo' + secondary.omnibus_config = <<~OMNIBUS + geo_secondary_role['enable'] = true; gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc'; OMNIBUS - primary.instance do - Component::Gitlab.perform do |secondary| - secondary.release = release - secondary.network = 'geo' - secondary.omnibus_config = <<~OMNIBUS - geo_secondary_role['enable'] = true; - gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc'; - OMNIBUS - - secondary.instance do - # shellout to instance specs - end - end + secondary.instance do + puts 'Secondary started!' end end + + # Component::Gitlab.perform do |primary| + # primary.release = release + # primary.network = 'geo' + # primary.omnibus_config = <<~OMNIBUS + # geo_primary_role['enable'] = true; + # postgresql['listen_address'] = '0.0.0.0'; + # postgresql['trust_auth_cidr_addresses'] = ['0.0.0.0/0','0.0.0.0/0']; + # postgresql['md5_auth_cidr_addresses'] = ['0.0.0.0/0']; + # postgresql['max_replication_slots'] = 1; + # gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc'; + # OMNIBUS + # + # primary.instance do + # Component::Gitlab.perform do |secondary| + # secondary.release = release + # secondary.network = 'geo' + # secondary.omnibus_config = <<~OMNIBUS + # geo_secondary_role['enable'] = true; + # gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc'; + # OMNIBUS + # + # secondary.instance do + # # shellout to instance specs + # end + # end + # end + # end end end end -- GitLab From cb091cd4c9b0fb5529e71dcf5452f6300ec52c24 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 13 Nov 2017 11:47:50 +0100 Subject: [PATCH 16/19] Try using GitLab Runner with more memory available --- .gitlab-ci.yml | 4 ++ .../qa/scenario/test/integration/geo.rb | 57 +++++++------------ 2 files changed, 26 insertions(+), 35 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 854775c98..baf5776c2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -98,6 +98,10 @@ before_script: ee:geo: script: bin/qa Test::Integration::Geo EE allow_failure: true + tags: + - docker + - 7gb + - triggered-packages <<: *test notify:slack: diff --git a/lib/gitlab/qa/scenario/test/integration/geo.rb b/lib/gitlab/qa/scenario/test/integration/geo.rb index 3f61ab1c0..e57580e55 100644 --- a/lib/gitlab/qa/scenario/test/integration/geo.rb +++ b/lib/gitlab/qa/scenario/test/integration/geo.rb @@ -11,46 +11,33 @@ module Gitlab raise ArgumentError, 'Geo is EE only!' end - Component::Gitlab.perform do |secondary| - secondary.release = release - secondary.network = 'geo' - secondary.omnibus_config = <<~OMNIBUS - geo_secondary_role['enable'] = true; + Component::Gitlab.perform do |primary| + primary.release = release + primary.network = 'geo' + primary.omnibus_config = <<~OMNIBUS + geo_primary_role['enable'] = true; + postgresql['listen_address'] = '0.0.0.0'; + postgresql['trust_auth_cidr_addresses'] = ['0.0.0.0/0','0.0.0.0/0']; + postgresql['md5_auth_cidr_addresses'] = ['0.0.0.0/0']; + postgresql['max_replication_slots'] = 1; gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc'; OMNIBUS - secondary.instance do - puts 'Secondary started!' + primary.instance do + Component::Gitlab.perform do |secondary| + secondary.release = release + secondary.network = 'geo' + secondary.omnibus_config = <<~OMNIBUS + geo_secondary_role['enable'] = true; + gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc'; + OMNIBUS + + secondary.instance do + puts 'Geo nodes configured!' + end + end end end - - # Component::Gitlab.perform do |primary| - # primary.release = release - # primary.network = 'geo' - # primary.omnibus_config = <<~OMNIBUS - # geo_primary_role['enable'] = true; - # postgresql['listen_address'] = '0.0.0.0'; - # postgresql['trust_auth_cidr_addresses'] = ['0.0.0.0/0','0.0.0.0/0']; - # postgresql['md5_auth_cidr_addresses'] = ['0.0.0.0/0']; - # postgresql['max_replication_slots'] = 1; - # gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc'; - # OMNIBUS - # - # primary.instance do - # Component::Gitlab.perform do |secondary| - # secondary.release = release - # secondary.network = 'geo' - # secondary.omnibus_config = <<~OMNIBUS - # geo_secondary_role['enable'] = true; - # gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc'; - # OMNIBUS - # - # secondary.instance do - # # shellout to instance specs - # end - # end - # end - # end end end end -- GitLab From 7ae329208e98d42f67c137490e92e58d30c4b800 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 13 Nov 2017 12:15:11 +0100 Subject: [PATCH 17/19] Do not wait for secondary to start correctly See gitlab-org/gitlab-ee#3999 --- lib/gitlab/qa/scenario/test/integration/geo.rb | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/qa/scenario/test/integration/geo.rb b/lib/gitlab/qa/scenario/test/integration/geo.rb index e57580e55..0a8974032 100644 --- a/lib/gitlab/qa/scenario/test/integration/geo.rb +++ b/lib/gitlab/qa/scenario/test/integration/geo.rb @@ -32,8 +32,17 @@ module Gitlab gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc'; OMNIBUS - secondary.instance do - puts 'Geo nodes configured!' + secondary.act do + # TODO, we do not wait for secondary to start because of + # https://gitlab.com/gitlab-org/gitlab-ee/issues/3999 + # + # rubocop:disable Style/Semicolon + prepare; start; reconfigure + + # shellout to instance specs + puts 'Running Geo primary / secondary specs!' + + teardown end end end -- GitLab From 9bc559d202afb60c6260ebfa98fa4cf7443f7a9b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 13 Nov 2017 12:26:34 +0100 Subject: [PATCH 18/19] Enable all pipeline jobs again after solving failures --- .gitlab-ci.yml | 132 ++++++++++++++++++++++++------------------------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index baf5776c2..35cdc1016 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,41 +16,41 @@ variables: before_script: - bundle install -# check:rubocop: -# stage: check -# image: $TEST_IMAGE -# script: bundle exec rubocop -# except: -# - triggers -# tags: -# - docker -# -# check:rspec: -# stage: check -# image: $TEST_IMAGE -# script: bundle exec rspec -# except: -# - triggers -# tags: -# - docker -# -# .sanity: &sanity -# stage: sanity -# image: $TEST_IMAGE -# except: -# - triggers -# tags: -# - docker -# -# ce:version: -# script: bin/qa Test::Sanity::Version CE -# allow_failure: true -# <<: *sanity -# -# ee:version: -# script: bin/qa Test::Sanity::Version EE -# allow_failure: true -# <<: *sanity +check:rubocop: + stage: check + image: $TEST_IMAGE + script: bundle exec rubocop + except: + - triggers + tags: + - docker + +check:rspec: + stage: check + image: $TEST_IMAGE + script: bundle exec rspec + except: + - triggers + tags: + - docker + +.sanity: &sanity + stage: sanity + image: $TEST_IMAGE + except: + - triggers + tags: + - docker + +ce:version: + script: bin/qa Test::Sanity::Version CE + allow_failure: true + <<: *sanity + +ee:version: + script: bin/qa Test::Sanity::Version EE + allow_failure: true + <<: *sanity .test: &test stage: test @@ -63,37 +63,37 @@ before_script: paths: - screenshots/*.png -# ce:instance: -# script: bin/qa Test::Instance::Image CE -# <<: *test -# -# ee:instance: -# script: bin/qa Test::Instance::Image EE -# <<: *test -# -# ce:image: -# script: bin/qa Test::Omnibus::Image CE -# <<: *test -# -# ee:image: -# script: bin/qa Test::Omnibus::Image EE -# <<: *test -# -# ce:upgrade: -# script: bin/qa Test::Omnibus::Upgrade CE -# <<: *test -# -# ee:upgrade: -# script: bin/qa Test::Omnibus::Upgrade EE -# <<: *test -# -# ce:mattermost: -# script: bin/qa Test::Integration::Mattermost CE -# <<: *test -# -# ee:mattermost: -# script: bin/qa Test::Integration::Mattermost EE -# <<: *test +ce:instance: + script: bin/qa Test::Instance::Image CE + <<: *test + +ee:instance: + script: bin/qa Test::Instance::Image EE + <<: *test + +ce:image: + script: bin/qa Test::Omnibus::Image CE + <<: *test + +ee:image: + script: bin/qa Test::Omnibus::Image EE + <<: *test + +ce:upgrade: + script: bin/qa Test::Omnibus::Upgrade CE + <<: *test + +ee:upgrade: + script: bin/qa Test::Omnibus::Upgrade EE + <<: *test + +ce:mattermost: + script: bin/qa Test::Integration::Mattermost CE + <<: *test + +ee:mattermost: + script: bin/qa Test::Integration::Mattermost EE + <<: *test ee:geo: script: bin/qa Test::Integration::Geo EE -- GitLab From 42bca3c0a9fb541b15ab0269a343fef697d9e61b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 13 Nov 2017 12:33:29 +0100 Subject: [PATCH 19/19] Trigger Geo instance specs after initial orchestration --- lib/gitlab/qa/scenario/test/integration/geo.rb | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/qa/scenario/test/integration/geo.rb b/lib/gitlab/qa/scenario/test/integration/geo.rb index 0a8974032..cfdc9a063 100644 --- a/lib/gitlab/qa/scenario/test/integration/geo.rb +++ b/lib/gitlab/qa/scenario/test/integration/geo.rb @@ -4,7 +4,11 @@ module Gitlab module Test module Integration class Geo < Scenario::Template - def perform(release) # rubocop:disable Metrics/MethodLength + ## + # rubocop:disable Metrics/MethodLength + # rubocop:disable Metrics/AbcSize + # + def perform(release) release = Release.new(release) unless release.edition == :ee @@ -42,6 +46,18 @@ module Gitlab # shellout to instance specs puts 'Running Geo primary / secondary specs!' + Component::Specs.perform do |specs| + specs.suite = 'Test::Integration::Geo' + specs.release = release + specs.network = 'geo' + specs.args = [ + '--primary-address', primary.address, + '--primary-name', primary.name, + '--secondary-address', secondary.address, + '--secondary-name', secondary.name + ] + end + teardown end end -- GitLab
RetroSearch is an open source project built by @garambo
| Open a GitHub Issue
Search and Browse the WWW like it's 1997 | Search results from DuckDuckGo
HTML:
3.2
| Encoding:
UTF-8
| Version:
0.7.4