RetroSearch Browse

Home - News ( United States | United Kingdom | Italy | Germany ) - Football scores

Showing content from https://github.com/orgrim/pg_back/commit/cfeb54d70ad206d1653c6a9a93af225b3cc89760 below:

Add an option to list files from remote locations · orgrim/pg_back@cfeb54d · GitHub

File tree Expand file treeCollapse file tree 4 files changed

+93

-18

lines changed

Filter options

Expand file treeCollapse file tree 4 files changed

+93

-18

lines changed Original file line number Diff line number Diff line change


 @@ -219,9 +219,12 @@ Previously uploaded files can be downloaded using the `--download` option with

219 219


 a value different than `none`, similarly to `--upload`. The options to setup

220 220


 the remote access are the same as `--upload`.

221 221 222

-
 When downloading files, dumps are not performed. Arguments on the commandline

223

-
 (database names when dumping) are used as shell globs to choose which files to

224

-
 the backup directory.

222

+
 It is possible to only list remote files with `--list-remote` with a value

223

+
 different than `none`, similarly to `--upload` and `--download`.

224 + 225

+
 When listing or downloading files, dumps are not performed. Arguments on the

226

+
 commandline (database names when dumping) are used as shell globs to

227

+
 select/filter files.

225 228 226 229


 If `--download` is used at the same time as `--decrypt`, files are downloaded

227 230


 first, then files matching globs are decrypted.

Original file line number Diff line number Diff line change


 @@ -84,6 +84,7 @@ type options struct {

84 84 85 85


 Upload string // values are none, s3, sftp, gcs

86 86


 Download string // values are none, s3, sftp, gcs

87

+
 ListRemote string // values are none, s3, sftp, gcs

87 88


 PurgeRemote bool

88 89


 S3Region string

89 90


 S3Bucket string


 @@ -134,6 +135,7 @@ func defaultOptions() options {

134 135


 WithRolePasswords: true,

135 136


 Upload: "none",

136 137


 Download: "none",

138

+
 ListRemote: "none",

137 139


 AzureEndpoint: "blob.core.windows.net",

138 140

139 141


 @@ -287,6 +289,7 @@ func parseCli(args []string) (options, []string, error) {

287 289 288 290


 pflag.StringVar(&opts.Upload, "upload", "none", "upload produced files to target (s3, gcs,..) use \"none\" to override\nconfiguration file and disable upload")

289 291


 pflag.StringVar(&opts.Download, "download", "none", "download files from target (s3, gcs,..) instead of dumping. DBNAMEs become\nglobs to select files")

292

+
 pflag.StringVar(&opts.ListRemote, "list-remote", "none", "list the remote files on s3, gcs, sftp, azure instead of dumping. DBNAMEs become\nglobs to select files")

290 293


 purgeRemote := pflag.String("purge-remote", "no", "purge the file on remote location after upload, with the same rules\nas the local directory")

291 294 292 295


 pflag.StringVar(&opts.S3Region, "s3-region", "", "S3 region")


 @@ -451,12 +454,16 @@ func parseCli(args []string) (options, []string, error) {

451 454


 return opts, changed, fmt.Errorf("invalid value for --download: %s", err)

452 455

453 456 457

+
 if err := validateEnum(opts.ListRemote, stores); err != nil {

458

+
 return opts, changed, fmt.Errorf("invalid value for --list-remote: %s", err)

459

+
 }

460 + 454 461


 opts.PurgeRemote, err = validateYesNoOption(*purgeRemote)

455 462


 if err != nil {

456 463


 return opts, changed, fmt.Errorf("invalid value for --purge-remote: %s", err)

457 464

458 465 459

-
 for _, o := range []string{opts.Upload, opts.Download} {

466

+
 for _, o := range []string{opts.Upload, opts.Download, opts.ListRemote} {

460 467


 switch o {

461 468


 case "s3":

462 469


 // Validate S3 options


 @@ -827,6 +834,8 @@ func mergeCliAndConfigOptions(cliOpts options, configOpts options, onCli []strin

827 834


 opts.Upload = cliOpts.Upload

828 835


 case "download":

829 836


 opts.Download = cliOpts.Download

837

+
 case "list-remote":

838

+
 opts.ListRemote = cliOpts.ListRemote

830 839


 case "purge-remote":

831 840


 opts.PurgeRemote = cliOpts.PurgeRemote

832 841 Original file line number Diff line number Diff line change


 @@ -196,6 +196,7 @@ func TestDefaultOptions(t *testing.T) {

196 196


 WithRolePasswords: true,

197 197


 Upload: "none",

198 198


 Download: "none",

199

+
 ListRemote: "none",

199 200


 AzureEndpoint: "blob.core.windows.net",

200 201

201 202


 @@ -240,6 +241,7 @@ func TestParseCli(t *testing.T) {

240 241


 WithRolePasswords: true,

241 242


 Upload: "none",

242 243


 Download: "none",

244

+
 ListRemote: "none",

243 245


 AzureEndpoint: "blob.core.windows.net",

244 246

},

245 247


 false,


 @@ -265,6 +267,7 @@ func TestParseCli(t *testing.T) {

265 267


 WithRolePasswords: true,

266 268


 Upload: "none",

267 269


 Download: "none",

270

+
 ListRemote: "none",

268 271


 AzureEndpoint: "blob.core.windows.net",

269 272

},

270 273


 false,


 @@ -315,6 +318,7 @@ func TestParseCli(t *testing.T) {

315 318


 WithRolePasswords: true,

316 319


 Upload: "wrong",

317 320


 Download: "none",

321

+
 ListRemote: "none",

318 322


 AzureEndpoint: "blob.core.windows.net",

319 323

},

320 324


 false,


 @@ -341,6 +345,7 @@ func TestParseCli(t *testing.T) {

341 345


 WithRolePasswords: true,

342 346


 Upload: "none",

343 347


 Download: "wrong",

348

+
 ListRemote: "none",

344 349


 AzureEndpoint: "blob.core.windows.net",

345 350

},

346 351


 false,


 @@ -375,6 +380,7 @@ func TestParseCli(t *testing.T) {

375 380


 WithRolePasswords: true,

376 381


 Upload: "none",

377 382


 Download: "none",

383

+
 ListRemote: "none",

378 384


 AzureEndpoint: "blob.core.windows.net",

379 385

},

380 386


 false,


 @@ -401,6 +407,7 @@ func TestParseCli(t *testing.T) {

401 407


 WithRolePasswords: true,

402 408


 Upload: "none",

403 409


 Download: "none",

410

+
 ListRemote: "none",

404 411


 AzureEndpoint: "blob.core.windows.net",

405 412

},

406 413


 false,


 @@ -427,6 +434,7 @@ func TestParseCli(t *testing.T) {

427 434


 WithRolePasswords: true,

428 435


 Upload: "none",

429 436


 Download: "none",

437

+
 ListRemote: "none",

430 438


 AzureEndpoint: "blob.core.windows.net",

431 439

},

432 440


 false,


 @@ -532,6 +540,7 @@ func TestLoadConfigurationFile(t *testing.T) {

532 540


 WithRolePasswords: true,

533 541


 Upload: "none",

534 542


 Download: "none",

543

+
 ListRemote: "none",

535 544


 AzureEndpoint: "blob.core.windows.net",

536 545

},

537 546

},


 @@ -554,6 +563,7 @@ func TestLoadConfigurationFile(t *testing.T) {

554 563


 WithRolePasswords: true,

555 564


 Upload: "none",

556 565


 Download: "none",

566

+
 ListRemote: "none",

557 567


 AzureEndpoint: "blob.core.windows.net",

558 568

},

559 569

},


 @@ -575,6 +585,7 @@ func TestLoadConfigurationFile(t *testing.T) {

575 585


 WithRolePasswords: true,

576 586


 Upload: "none",

577 587


 Download: "none",

588

+
 ListRemote: "none",

578 589


 AzureEndpoint: "blob.core.windows.net",

579 590

},

580 591

},


 @@ -596,6 +607,7 @@ func TestLoadConfigurationFile(t *testing.T) {

596 607


 WithRolePasswords: true,

597 608


 Upload: "none",

598 609


 Download: "none",

610

+
 ListRemote: "none",

599 611


 AzureEndpoint: "blob.core.windows.net",

600 612

},

601 613

},


 @@ -646,6 +658,7 @@ func TestLoadConfigurationFile(t *testing.T) {

646 658


 WithRolePasswords: true,

647 659


 Upload: "none",

648 660


 Download: "none",

661

+
 ListRemote: "none",

649 662


 AzureEndpoint: "blob.core.windows.net",

650 663

},

651 664

},


 @@ -688,6 +701,7 @@ func TestLoadConfigurationFile(t *testing.T) {

688 701


 WithRolePasswords: false,

689 702


 Upload: "none",

690 703


 Download: "none",

704

+
 ListRemote: "none",

691 705


 AzureEndpoint: "blob.core.windows.net",

692 706

},

693 707

},


 @@ -755,6 +769,7 @@ func TestMergeCliAndConfigoptions(t *testing.T) {

755 769


 WithRolePasswords: true,

756 770


 Upload: "none",

757 771


 Download: "none",

772

+
 ListRemote: "none",

758 773


 AzureEndpoint: "blob.core.windows.net",

759 774

760 775 Original file line number Diff line number Diff line change


 @@ -173,32 +173,42 @@ func run() (retVal error) {

173 173


 return fmt.Errorf("required cipher parameters not present: %w", err)

174 174

175 175 176

-
 if (opts.Upload == "s3" || opts.Download == "s3") && opts.S3Bucket == "" {

176

+
 if (opts.Upload == "s3" || opts.Download == "s3" || opts.ListRemote == "s3") && opts.S3Bucket == "" {

177 177


 return fmt.Errorf("a bucket is mandatory with s3")

178 178

179 179 180

-
 if (opts.Upload == "gcs" || opts.Download == "gcs") && opts.GCSBucket == "" {

180

+
 if (opts.Upload == "gcs" || opts.Download == "gcs" || opts.ListRemote == "gcs") && opts.GCSBucket == "" {

181 181


 return fmt.Errorf("a bucket is mandatory with gcs")

182 182

183 183 184

-
 if (opts.Upload == "azure" || opts.Download == "azure") && opts.AzureContainer == "" {

184

+
 if (opts.Upload == "azure" || opts.Download == "azure" || opts.ListRemote == "azure") && opts.AzureContainer == "" {

185 185


 return fmt.Errorf("a container is mandatory with azure")

186 186

187 187 188

+
 // Run actions that won't dump databases first, in that case the list

189

+
 // of databases become file globs. Avoid getting wrong globs from the

190

+
 // config file since we are using the remaining args from the command

191

+
 // line that are usually as a list of databases to dump

192

+
 globs := []string{}

193

+
 for _, v := range cliOptList {

194

+
 if v == "include-dbs" {

195

+
 globs = opts.Dbnames

196

+
 break

197

+
 }

198

+
 }

199 + 200

+
 // Listing remote files take priority over the other options that won't dump databases

201

+
 if opts.ListRemote != "none" {

202

+
 if err := listRemoteFiles(opts.ListRemote, opts, globs); err != nil {

203

+
 return err

204

+
 }

205 + 206

+
 return nil

207

+
 }

208 + 188 209


 // When asked to download or decrypt the backups, do it here and exit, we have all

189 210


 // required input (passphrase and backup directory)

190 211


 if opts.Decrypt || opts.Download != "none" {

191

-
 // Avoid getting wrong globs from the config file since we are

192

-
 // using the remaining args from the command line that are

193

-
 // usually as a list of databases to dump

194

-
 globs := []string{}

195

-
 for _, v := range cliOptList {

196

-
 if v == "include-dbs" {

197

-
 globs = opts.Dbnames

198

-
 break

199

-
 }

200

-
 }

201 - 202 212


 if opts.Download != "none" {

203 213


 if err := downloadFiles(opts.Download, opts, opts.Directory, globs); err != nil {

204 214


 return err


 @@ -1004,6 +1014,44 @@ func dumpConfigFiles(dir string, timeFormat string, db *pg, fc chan<- sumFileJob

1004 1014


 return nil

1005 1015

1006 1016 1017

+
 func listRemoteFiles(repoName string, opts options, globs []string) error {

1018

+
 repo, err := NewRepo(repoName, opts)

1019

+
 if err != nil {

1020

+
 return err

1021

+
 }

1022 + 1023

+
 remoteFiles, err := repo.List("")

1024

+
 if err != nil {

1025

+
 return fmt.Errorf("could not list contents of remote location: %w", err)

1026

+
 }

1027 + 1028

+
 for _, i := range remoteFiles {

1029

+
 keep := false

1030

+
 if len(globs) == 0 {

1031

+
 keep = true

1032

+
 }

1033 + 1034

+
 for _, glob := range globs {

1035

+
 keep, err = filepath.Match(glob, i.key)

1036

+
 if err != nil {

1037

+
 return fmt.Errorf("bad patern: %w", err)

1038

+
 }

1039 + 1040

+
 if keep {

1041

+
 break

1042

+
 }

1043

+
 }

1044 + 1045

+
 if !keep {

1046

+
 continue

1047

+
 }

1048 + 1049

+
 fmt.Println(i.key)

1050

+
 }

1051 + 1052

+
 return nil

1053

+
 }

1054 + 1007 1055


 func downloadFiles(repoName string, opts options, dir string, globs []string) error {

1008 1056


 repo, err := NewRepo(repoName, opts)

1009 1057


 if err != nil {

You can’t perform that action at this time.

RetroSearch is an open source project built by @garambo | Open a GitHub Issue

Search and Browse the WWW like it's 1997 | Search results from DuckDuckGo

HTML: 3.2 | Encoding: UTF-8 | Version: 0.7.4