From 65e8aea36e493fe89ab1a32bcaefe6d906db5e2e Mon Sep 17 00:00:00 2001 From: Jianchao Ma Date: Fri, 29 Sep 2023 14:29:33 +0000 Subject: [PATCH 01/16] =?UTF-8?q?feat:=20pr=C3=A9paration=20des=20fichiers?= =?UTF-8?q?=20pour=20int=C3=A9gration=20de=20xixo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/pimo/main.go | 9 +++++++++ .../go.mod => exemple_expected.xml | 0 go.mod | 2 ++ go.sum | 5 +++++ internal/app/pimo/xixo.go | 11 +++++++++++ test/exemple.xml | 19 +++++++++++++++++++ test/masking_account.yml | 15 +++++++++++++++ test/masking_agency.yml | 8 ++++++++ 8 files changed, 69 insertions(+) rename web/play/node_modules/go.mod => exemple_expected.xml (100%) create mode 100644 internal/app/pimo/xixo.go create mode 100644 test/exemple.xml create mode 100644 test/masking_account.yml create mode 100644 test/masking_agency.yml diff --git a/cmd/pimo/main.go b/cmd/pimo/main.go index 75385fdb..1c2209ca 100644 --- a/cmd/pimo/main.go +++ b/cmd/pimo/main.go @@ -67,6 +67,7 @@ var ( statsTemplate string statsDestinationEnv = os.Getenv("PIMO_STATS_URL") statsTemplateEnv = os.Getenv("PIMO_STATS_TEMPLATE") + subscribers map[string]string ) func main() { @@ -120,6 +121,14 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa }, }) + xmlCmd := &cobra.Command{ + Use: "xml", + Run: func(cmd *cobra.Command, args []string) { + pimo.ParseXML(cmd.InOrStdin(), cmd.OutOrStdout()) + }, + } + xmlCmd.Flags().StringToStringVar(&subscribers, "subscribers", map[string]string{}, "element where apply mask") + rootCmd.AddCommand(&cobra.Command{ Use: "flow", Run: func(cmd *cobra.Command, args []string) { diff --git a/web/play/node_modules/go.mod b/exemple_expected.xml similarity index 100% rename from web/play/node_modules/go.mod rename to exemple_expected.xml diff --git a/go.mod b/go.mod index 69959ef8..192e5ab2 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/spf13/cast v1.5.1 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 + github.com/youen/xixo v0.0.0-20230922162717-6d8c927577cb github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea gitlab.com/c0b/go-ordered-json v0.0.0-20201030195603-febf46534d5a golang.org/x/text v0.12.0 @@ -40,6 +41,7 @@ require ( github.com/shopspring/decimal v1.2.0 // indirect github.com/smartystreets/goconvey v1.6.4 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/tamerh/xpath v1.0.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect golang.org/x/crypto v0.11.0 // indirect diff --git a/go.sum b/go.sum index 520fda7b..0c1b19bd 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,7 @@ github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/adrienaury/zeromdc v0.0.0-20221116212822-6a366c26ee61 h1:GXmgCyG0oDr3BFL+NYcwKEP2H3L9DuBo66ZR++zNasA= github.com/adrienaury/zeromdc v0.0.0-20221116212822-6a366c26ee61/go.mod h1:5UlMlw0MRjEAms20gDadR5GrN2wEp9XEXTBiMp/XI4E= +github.com/antchfx/xpath v1.2.4 h1:dW1HB/JxKvGtJ9WyVGJ0sIoEcqftV3SqIstujI+B9XY= github.com/capitalone/fpe v1.2.1 h1:/r81KhhTkfmxjjr2HKr+WYTLrMjPnn0gtK/L8gKNfts= github.com/capitalone/fpe v1.2.1/go.mod h1:hI6YzL2v2WkosaevH24sYHyyDAzacfqkpaOYc/0Qn7g= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -95,11 +96,15 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tamerh/xpath v1.0.0 h1:NccMES/Ej8slPCFDff73Kf6V1xu9hdbuKf2RyDsxf5Q= +github.com/tamerh/xpath v1.0.0/go.mod h1:t0wnh72FQlOVEO20f2Dl3EoVxso9GnLREh1WTpvNmJQ= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/youen/xixo v0.0.0-20230922162717-6d8c927577cb h1:FpW08PivlO+zqpU92KxbS0jkvsiB447eAyYSfJqsn/Q= +github.com/youen/xixo v0.0.0-20230922162717-6d8c927577cb/go.mod h1:Nr//UqY2ngqBw5P+HkiYhppjS3dmqlZ9ENKDXtFrTsM= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea h1:CyhwejzVGvZ3Q2PSbQ4NRRYn+ZWv5eS1vlaEusT+bAI= github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea/go.mod h1:eNr558nEUjP8acGw8FFjTeWvSgU1stO7FAO6eknhHe4= diff --git a/internal/app/pimo/xixo.go b/internal/app/pimo/xixo.go new file mode 100644 index 00000000..55feedde --- /dev/null +++ b/internal/app/pimo/xixo.go @@ -0,0 +1,11 @@ +package pimo + +import ( + "io" + + "github.com/youen/xixo/pkg/xixo" +) + +func ParseXML(input io.Reader, output io.Writer) { + xixo.NewXMLParser(input, output) +} diff --git a/test/exemple.xml b/test/exemple.xml new file mode 100644 index 00000000..a7d4d2af --- /dev/null +++ b/test/exemple.xml @@ -0,0 +1,19 @@ + + + + Nantes Agency + 0032 + + + Doe John + 12345 + 50000 + 10000 + + + Smith Jane + 67890 + 60000 + 12000 + + diff --git a/test/masking_account.yml b/test/masking_account.yml new file mode 100644 index 00000000..21ad61b6 --- /dev/null +++ b/test/masking_account.yml @@ -0,0 +1,15 @@ +version: "1" +seed: 42 + +masking: + - selector: + jsonpath: "name" + mask: + randomChoice: + - "Mickael" + - "Mathieu " + - "Marcelle" + - selector: + jsonpath: "account_number" + mask: + regex: "d{5}$" diff --git a/test/masking_agency.yml b/test/masking_agency.yml new file mode 100644 index 00000000..aaf9f126 --- /dev/null +++ b/test/masking_agency.yml @@ -0,0 +1,8 @@ +version: "1" +seed: 42 + +masking: + - selector: + jsonpath: "agency_number" + mask: + regex: "d{4}$" From c2805a7bf564bcd2a68454dcdaec3c36ce96ca82 Mon Sep 17 00:00:00 2001 From: Jianchao Ma Date: Fri, 13 Oct 2023 13:05:55 +0000 Subject: [PATCH 02/16] =?UTF-8?q?feat:traitement=20xml=20v1=20avec=20un=20?= =?UTF-8?q?probl=C3=A8me=20de=20mask=20identique=20=C3=A0=20r=C3=A9gler?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/pimo/main.go | 56 +++++++++++++++++++++++++++++++--- exemple_expected.xml | 0 internal/app/pimo/pimo.go | 43 ++++++++++++++++++++++++++ internal/app/pimo/pimo_test.go | 23 ++++++++++++++ internal/app/pimo/xixo.go | 21 +++++++++++-- test/masking_account.yml | 4 +-- test/masking_agency.yml | 2 +- 7 files changed, 139 insertions(+), 10 deletions(-) delete mode 100644 exemple_expected.xml diff --git a/cmd/pimo/main.go b/cmd/pimo/main.go index 1c2209ca..e6336f71 100644 --- a/cmd/pimo/main.go +++ b/cmd/pimo/main.go @@ -67,7 +67,7 @@ var ( statsTemplate string statsDestinationEnv = os.Getenv("PIMO_STATS_URL") statsTemplateEnv = os.Getenv("PIMO_STATS_TEMPLATE") - subscribers map[string]string + subscriberName map[string]string ) func main() { @@ -120,14 +120,60 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa fmt.Println(jsonschema) }, }) - + // Add command for XML transformer xmlCmd := &cobra.Command{ - Use: "xml", + Use: "xml", + Short: "Transform fichier XML with substitution", Run: func(cmd *cobra.Command, args []string) { - pimo.ParseXML(cmd.InOrStdin(), cmd.OutOrStdout()) + initLog() + if len(catchErrors) > 0 { + skipLineOnError = true + skipLogFile = catchErrors + } + config := pimo.Config{ + EmptyInput: emptyInput, + RepeatUntil: repeatUntil, + RepeatWhile: repeatWhile, + Iteration: iteration, + SkipLineOnError: skipLineOnError, + SkipFieldOnError: skipFieldOnError, + SkipLogFile: skipLogFile, + CachesToDump: cachesToDump, + CachesToLoad: cachesToLoad, + } + // ces étapes doivent retrouver dans xixo.go pour ne pas alourdre le main.gopi + parser := pimo.ParseXML(cmd.InOrStdin(), cmd.OutOrStdout()) + // Map the command line balise name to fit the masking configuration + for elementName, mask := range subscriberName { + pdef, err := model.LoadPipelineDefinitionFromFile(mask) + if err != nil { + fmt.Printf("Error when charging pipeline for %s : %v\n", elementName, err) + return + } + ctx := pimo.NewContext(pdef) + if err := ctx.Configure(config); err != nil { + log.Err(err).Msg("Cannot configure pipeline") + log.Warn().Int("return", 1).Msg("End PIMO") + os.Exit(1) + } + + parser.RegisterMapCallback(elementName, func(m map[string]string) (map[string]string, error) { + transformedData, err := ctx.ExecuteMap(m) + if err != nil { + return nil, err + } + // Update m with masked data + for key, value := range transformedData { + m[key] = value + } + return m, nil + }) + } + parser.Stream() }, } - xmlCmd.Flags().StringToStringVar(&subscribers, "subscribers", map[string]string{}, "element where apply mask") + xmlCmd.Flags().StringToStringVar(&subscriberName, "subscriber", map[string]string{}, "name of element to mask") + rootCmd.AddCommand(xmlCmd) rootCmd.AddCommand(&cobra.Command{ Use: "flow", diff --git a/exemple_expected.xml b/exemple_expected.xml deleted file mode 100644 index e69de29b..00000000 diff --git a/internal/app/pimo/pimo.go b/internal/app/pimo/pimo.go index 0ae72e46..6ca56d78 100755 --- a/internal/app/pimo/pimo.go +++ b/internal/app/pimo/pimo.go @@ -331,3 +331,46 @@ func updateContext(counter int) { context := over.MDC().GetString("context") over.MDC().Set("context", re.ReplaceAllString(context, fmt.Sprintf("[%d]", counter))) } + +func (ctx *Context) ExecuteMap(data map[string]string) (map[string]string, error) { + input := model.NewDictionary() + + for k, v := range data { + input = input.With(k, v) + } + + cfg := Config{ + EmptyInput: false, + Iteration: 1, + SingleInput: &input, + } + + err := ctx.Configure(cfg) + if err != nil { + return nil, err + } + result := []model.Entry{} + err = ctx.pipeline.AddSink(model.NewSinkToSlice(&result)).Run() + if err != nil { + return nil, err + } + + newData := make(map[string]string) + + if len(result) > 0 { + new_map, ok := result[0].(model.Dictionary) + if !ok { + return nil, fmt.Errorf("result is not Dictionary") + } + unordered := new_map.Unordered() + for k, v := range unordered { + stringValue, ok := v.(string) + if !ok { + return nil, fmt.Errorf("Result is not a string") + } + newData[k] = stringValue + } + return newData, nil + } + return nil, fmt.Errorf("Result is not a map[string]string") +} diff --git a/internal/app/pimo/pimo_test.go b/internal/app/pimo/pimo_test.go index e61ccae8..8e685b0e 100755 --- a/internal/app/pimo/pimo_test.go +++ b/internal/app/pimo/pimo_test.go @@ -318,3 +318,26 @@ func LoadJsonLineFromDocument(filename string) (model.Dictionary, error) { // return jsonline.JSONToDictionary(compactLine.Bytes()) } + +func TestExecuteMap(t *testing.T) { + definition := model.Definition{ + Version: "1", + Masking: []model.Masking{ + { + Selector: model.SelectorType{Jsonpath: "name"}, + Mask: model.MaskType{ + HashInURI: "pimo://nameFR", + }, + }, + }, + } + + ctx := pimo.NewContext(definition) + + data := map[string]string{"name": "John"} + + newData, err := ctx.ExecuteMap(data) + + assert.Nil(t, err) + assert.NotEqual(t, "John", newData["name"]) +} diff --git a/internal/app/pimo/xixo.go b/internal/app/pimo/xixo.go index 55feedde..37152170 100644 --- a/internal/app/pimo/xixo.go +++ b/internal/app/pimo/xixo.go @@ -1,3 +1,20 @@ +// Copyright (C) 2022 CGI France +// +// This file is part of PIMO. +// +// PIMO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// PIMO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with PIMO. If not, see . + package pimo import ( @@ -6,6 +23,6 @@ import ( "github.com/youen/xixo/pkg/xixo" ) -func ParseXML(input io.Reader, output io.Writer) { - xixo.NewXMLParser(input, output) +func ParseXML(input io.Reader, output io.Writer) *xixo.XMLParser { + return xixo.NewXMLParser(input, output).EnableXpath() } diff --git a/test/masking_account.yml b/test/masking_account.yml index 21ad61b6..da40cb02 100644 --- a/test/masking_account.yml +++ b/test/masking_account.yml @@ -7,9 +7,9 @@ masking: mask: randomChoice: - "Mickael" - - "Mathieu " + - "Mathieu" - "Marcelle" - selector: jsonpath: "account_number" mask: - regex: "d{5}$" + regex: "[0-9]{5}$" diff --git a/test/masking_agency.yml b/test/masking_agency.yml index aaf9f126..6e4aac49 100644 --- a/test/masking_agency.yml +++ b/test/masking_agency.yml @@ -5,4 +5,4 @@ masking: - selector: jsonpath: "agency_number" mask: - regex: "d{4}$" + template: '{{MaskRegex "[0-9]{4}$"}}' From e47f4df7c5d6060e1feef7a93720f68fdc721c9b Mon Sep 17 00:00:00 2001 From: Jianchao Ma Date: Mon, 16 Oct 2023 13:05:34 +0000 Subject: [PATCH 03/16] feat: TestCallableMapSource --- exemple_expected.xml | 1 + pkg/model/model.go | 28 ++++++++++++++++++++++++++++ pkg/model/model_test.go | 5 +++++ test/exemple.xml | 6 ++++++ test/exemple_expected.xml | 39 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 79 insertions(+) create mode 100644 exemple_expected.xml create mode 100644 test/exemple_expected.xml diff --git a/exemple_expected.xml b/exemple_expected.xml new file mode 100644 index 00000000..3f10848e --- /dev/null +++ b/exemple_expected.xml @@ -0,0 +1 @@ +Error when charging pipeline for agency : open masking_agency.yml: no such file or directory diff --git a/pkg/model/model.go b/pkg/model/model.go index d5283a35..268cf3bb 100755 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -668,3 +668,31 @@ func NewSeeder(sourceField string, seed int64) Seeder { } return seeder } + +func NewCallableMapSource() Source { + return &CallableMapSource{} +} + +type CallableMapSource struct { + dictionaries []Dictionary + offset int +} + +func (source *CallableMapSource) Open() error { + source.offset = 0 + return nil +} + +func (source *CallableMapSource) Next() bool { + result := source.offset < len(source.dictionaries) + source.offset++ + return result +} + +func (source *CallableMapSource) Value() Entry { + return source.dictionaries[source.offset-1] +} + +func (source *CallableMapSource) Err() error { + return nil +} diff --git a/pkg/model/model_test.go b/pkg/model/model_test.go index 4d3ac6c0..5d07aa25 100755 --- a/pkg/model/model_test.go +++ b/pkg/model/model_test.go @@ -580,3 +580,8 @@ func TestCacheShouldProvide(t *testing.T) { assert.Equal(t, wanted, result) } + +func TestCallableMapSource(t *testing.T) { + source := NewCallableMapSource() + assert.NotNil(t, source) +} diff --git a/test/exemple.xml b/test/exemple.xml index a7d4d2af..aa3187b4 100644 --- a/test/exemple.xml +++ b/test/exemple.xml @@ -16,4 +16,10 @@ 60000 12000 + + Hello world + 00000 + 60000 + 12000 + diff --git a/test/exemple_expected.xml b/test/exemple_expected.xml new file mode 100644 index 00000000..ce433c8d --- /dev/null +++ b/test/exemple_expected.xml @@ -0,0 +1,39 @@ +contextSourceBefor +contextSourceafter &{[{0xc000128870}] 0} +data: {"name":"Doe John","account_number":"12345","annual_income":"50000","total_taxes_paid":"10000"} +singleInput: {"name":"Doe John","account_number":"12345","annual_income":"50000","total_taxes_paid":"10000"} +contextSourceBefor +contextSourceafter &{[{0xc000128bb8}] 0} +data: {"total_taxes_paid":"12000","name":"Smith Jane","account_number":"67890","annual_income":"60000"} +singleInput: {"total_taxes_paid":"12000","name":"Smith Jane","account_number":"67890","annual_income":"60000"} +contextSourceBefor +contextSourceafter &{[{0xc000129008}] 0} +data: {"account_number":"00000","annual_income":"60000","total_taxes_paid":"12000","name":"Hello world"} +singleInput: {"account_number":"00000","annual_income":"60000","total_taxes_paid":"12000","name":"Hello world"} +contextSourceBefor +contextSourceafter &{[{0xc000129458}] 0} + + + + Nantes Agency + 0032 + + + Mathieu + 18291 + 50000 + 10000 + + + Mathieu + 18291 + 60000 + 12000 + + + Mathieu + 18291 + 60000 + 12000 + + From 3610b609cbfad049f4d16522e325c9e32453ad10 Mon Sep 17 00:00:00 2001 From: Jianchao Ma Date: Mon, 16 Oct 2023 13:19:34 +0000 Subject: [PATCH 04/16] feat: add setValue to callableMapSource --- pkg/model/model.go | 16 ++++++++-------- pkg/model/model_test.go | 11 +++++++++++ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/pkg/model/model.go b/pkg/model/model.go index 268cf3bb..447d8fe3 100755 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -669,28 +669,28 @@ func NewSeeder(sourceField string, seed int64) Seeder { return seeder } -func NewCallableMapSource() Source { +func NewCallableMapSource() *CallableMapSource { return &CallableMapSource{} } type CallableMapSource struct { - dictionaries []Dictionary - offset int + value Entry } func (source *CallableMapSource) Open() error { - source.offset = 0 return nil } func (source *CallableMapSource) Next() bool { - result := source.offset < len(source.dictionaries) - source.offset++ - return result + return source.value != nil +} + +func (source *CallableMapSource) SetValue(value Entry) { + source.value = value } func (source *CallableMapSource) Value() Entry { - return source.dictionaries[source.offset-1] + return source.value } func (source *CallableMapSource) Err() error { diff --git a/pkg/model/model_test.go b/pkg/model/model_test.go index 5d07aa25..fd7717b4 100755 --- a/pkg/model/model_test.go +++ b/pkg/model/model_test.go @@ -585,3 +585,14 @@ func TestCallableMapSource(t *testing.T) { source := NewCallableMapSource() assert.NotNil(t, source) } + +func TestCallableMapSourceNextShouldReturnFalseBeforeValueIsSetted(t *testing.T) { + source := NewCallableMapSource() + assert.False(t, source.Next()) +} + +func TestCallableMapSourceNextShouldReturnTrueAfterValueIsSetted(t *testing.T) { + source := NewCallableMapSource() + source.SetValue(NewDictionary()) + assert.True(t, source.Next()) +} From 6fc91e142cc1d73ba15d3838d543b21d2a1adf8f Mon Sep 17 00:00:00 2001 From: Jianchao Ma Date: Mon, 16 Oct 2023 14:06:40 +0000 Subject: [PATCH 05/16] feat: XML version 1 done --- cmd/pimo/main.go | 6 +++++- exemple_expected.xml | 2 +- internal/app/pimo/pimo.go | 20 +++++++++--------- internal/app/pimo/pimo_test.go | 37 +++++++++++++++++++++++++++++++++- pkg/model/model.go | 7 +++++-- pkg/model/model_test.go | 7 +++++++ test/exemple_expected.xml | 26 ++++++------------------ test/masking_account.yml | 12 +++++------ 8 files changed, 75 insertions(+), 42 deletions(-) diff --git a/cmd/pimo/main.go b/cmd/pimo/main.go index e6336f71..72a88435 100644 --- a/cmd/pimo/main.go +++ b/cmd/pimo/main.go @@ -140,6 +140,7 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa SkipLogFile: skipLogFile, CachesToDump: cachesToDump, CachesToLoad: cachesToLoad, + Callback: true, } // ces étapes doivent retrouver dans xixo.go pour ne pas alourdre le main.gopi parser := pimo.ParseXML(cmd.InOrStdin(), cmd.OutOrStdout()) @@ -169,7 +170,10 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa return m, nil }) } - parser.Stream() + err := parser.Stream() + if err != nil { + log.Err(err).Msg("Error during parsing XML document") + } }, } xmlCmd.Flags().StringToStringVar(&subscriberName, "subscriber", map[string]string{}, "name of element to mask") diff --git a/exemple_expected.xml b/exemple_expected.xml index 3f10848e..615d883f 100644 --- a/exemple_expected.xml +++ b/exemple_expected.xml @@ -1 +1 @@ -Error when charging pipeline for agency : open masking_agency.yml: no such file or directory +Error when charging pipeline for account : open masking_account.yml: no such file or directory diff --git a/internal/app/pimo/pimo.go b/internal/app/pimo/pimo.go index 6ca56d78..e0e6cb4d 100755 --- a/internal/app/pimo/pimo.go +++ b/internal/app/pimo/pimo.go @@ -77,6 +77,7 @@ type Config struct { SkipLogFile string CachesToDump map[string]string CachesToLoad map[string]string + Callback bool } type Context struct { @@ -107,6 +108,9 @@ func (ctx *Context) Configure(cfg Config) error { over.AddGlobalFields("context") switch { + case cfg.Callback: + over.MDC().Set("context", "callback-input") + ctx.source = model.NewCallableMapSource() case cfg.EmptyInput: over.MDC().Set("context", "empty-input") ctx.source = model.NewSourceFromSlice([]model.Dictionary{model.NewPackedDictionary()}) @@ -338,19 +342,13 @@ func (ctx *Context) ExecuteMap(data map[string]string) (map[string]string, error for k, v := range data { input = input.With(k, v) } - - cfg := Config{ - EmptyInput: false, - Iteration: 1, - SingleInput: &input, - } - - err := ctx.Configure(cfg) - if err != nil { - return nil, err + source, ok := ctx.source.(*model.CallableMapSource) + if !ok { + return nil, fmt.Errorf("Source is not CallableMapSource") } + source.SetValue(input) result := []model.Entry{} - err = ctx.pipeline.AddSink(model.NewSinkToSlice(&result)).Run() + err := ctx.pipeline.AddSink(model.NewSinkToSlice(&result)).Run() if err != nil { return nil, err } diff --git a/internal/app/pimo/pimo_test.go b/internal/app/pimo/pimo_test.go index 8e685b0e..9dac75ee 100755 --- a/internal/app/pimo/pimo_test.go +++ b/internal/app/pimo/pimo_test.go @@ -331,8 +331,14 @@ func TestExecuteMap(t *testing.T) { }, }, } - ctx := pimo.NewContext(definition) + cfg := pimo.Config{ + Iteration: 1, + Callback: true, + } + + err := ctx.Configure(cfg) + assert.Nil(t, err) data := map[string]string{"name": "John"} @@ -341,3 +347,32 @@ func TestExecuteMap(t *testing.T) { assert.Nil(t, err) assert.NotEqual(t, "John", newData["name"]) } + +func Test2BaliseIdentity(t *testing.T) { + definition := model.Definition{ + Version: "1", + Seed: 42, + Masking: []model.Masking{ + { + Selector: model.SelectorType{Jsonpath: "name"}, + Mask: model.MaskType{ + RandomChoiceInURI: "pimo://nameFR", + }, + }, + }, + } + ctx := pimo.NewContext(definition) + cfg := pimo.Config{ + Iteration: 1, + Callback: true, + } + + err := ctx.Configure(cfg) + assert.Nil(t, err) + + data := map[string]string{"name": "John"} + newData1, err := ctx.ExecuteMap(data) + newData2, err := ctx.ExecuteMap(data) + assert.Nil(t, err) + assert.NotEqual(t, newData2["name"], newData1["name"]) +} diff --git a/pkg/model/model.go b/pkg/model/model.go index 447d8fe3..9d9c024c 100755 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -674,7 +674,8 @@ func NewCallableMapSource() *CallableMapSource { } type CallableMapSource struct { - value Entry + value Entry + nextValue Entry } func (source *CallableMapSource) Open() error { @@ -682,11 +683,13 @@ func (source *CallableMapSource) Open() error { } func (source *CallableMapSource) Next() bool { + source.value = source.nextValue + source.nextValue = nil return source.value != nil } func (source *CallableMapSource) SetValue(value Entry) { - source.value = value + source.nextValue = value } func (source *CallableMapSource) Value() Entry { diff --git a/pkg/model/model_test.go b/pkg/model/model_test.go index fd7717b4..0069d52c 100755 --- a/pkg/model/model_test.go +++ b/pkg/model/model_test.go @@ -596,3 +596,10 @@ func TestCallableMapSourceNextShouldReturnTrueAfterValueIsSetted(t *testing.T) { source.SetValue(NewDictionary()) assert.True(t, source.Next()) } + +func TestCallableMapSourceNextShouldReturnFalseAfterNextCalledTwice(t *testing.T) { + source := NewCallableMapSource() + source.SetValue(NewDictionary()) + assert.True(t, source.Next()) + assert.False(t, source.Next()) +} diff --git a/test/exemple_expected.xml b/test/exemple_expected.xml index ce433c8d..df1bef44 100644 --- a/test/exemple_expected.xml +++ b/test/exemple_expected.xml @@ -1,17 +1,3 @@ -contextSourceBefor -contextSourceafter &{[{0xc000128870}] 0} -data: {"name":"Doe John","account_number":"12345","annual_income":"50000","total_taxes_paid":"10000"} -singleInput: {"name":"Doe John","account_number":"12345","annual_income":"50000","total_taxes_paid":"10000"} -contextSourceBefor -contextSourceafter &{[{0xc000128bb8}] 0} -data: {"total_taxes_paid":"12000","name":"Smith Jane","account_number":"67890","annual_income":"60000"} -singleInput: {"total_taxes_paid":"12000","name":"Smith Jane","account_number":"67890","annual_income":"60000"} -contextSourceBefor -contextSourceafter &{[{0xc000129008}] 0} -data: {"account_number":"00000","annual_income":"60000","total_taxes_paid":"12000","name":"Hello world"} -singleInput: {"account_number":"00000","annual_income":"60000","total_taxes_paid":"12000","name":"Hello world"} -contextSourceBefor -contextSourceafter &{[{0xc000129458}] 0} @@ -19,20 +5,20 @@ contextSourceafter &{[{0xc000129458}] 0} 0032 - Mathieu - 18291 + Rolande + 1 50000 10000 - Mathieu - 18291 + Matéo + 2 60000 12000 - Mathieu - 18291 + Rosalie + 3 60000 12000 diff --git a/test/masking_account.yml b/test/masking_account.yml index da40cb02..101b3e32 100644 --- a/test/masking_account.yml +++ b/test/masking_account.yml @@ -5,11 +5,11 @@ masking: - selector: jsonpath: "name" mask: - randomChoice: - - "Mickael" - - "Mathieu" - - "Marcelle" + randomChoiceInUri: "pimo://nameFR" - selector: jsonpath: "account_number" - mask: - regex: "[0-9]{5}$" + masks: + - incremental: + start: 1 + increment: 1 + - template: "{{.account_number}}" From c9f4a42d5894bc9ef009c5ddc99298b1a5612a28 Mon Sep 17 00:00:00 2001 From: Jianchao Ma Date: Wed, 18 Oct 2023 08:43:37 +0000 Subject: [PATCH 06/16] feat: ajouter les documentations et test yaml --- CHANGELOG.md | 4 ++ README.md | 102 +++++++++++++++++++++++++++++++++ cmd/pimo/main.go | 16 ++---- exemple_expected.xml | 1 - internal/app/pimo/pimo.go | 4 +- internal/app/pimo/pimo_test.go | 9 +-- test/exemple_expected.xml | 6 +- 7 files changed, 122 insertions(+), 20 deletions(-) delete mode 100644 exemple_expected.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index f5252fc7..8e5b8e51 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ Types of changes - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. +## [1.20.0] + +- `Added` new features for parsing and masking XML files using the following command: `cat XMLfile | pimo xml --subscriber = > outputXMLfile`. This feature supports all level 1 elements that are not arrays. + ## [1.19.0] - `Added` new features for ff1 mask : `domain`, `preserve` and `onError`. diff --git a/README.md b/README.md index cab7e72f..4a97a3f0 100755 --- a/README.md +++ b/README.md @@ -955,6 +955,108 @@ By default, if not specified otherwise, these classes will be used (input -> out [Return to list of masks](#possible-masks) + +### Parsing XML files + +To use PIMO to masking data in an XML file, use in the following way : + +```bash + `cat data.xml | pimo xml --subscriber parentTagName=MaskName.yml > maskedData.xml` +``` + +Pimo selects specific tags within a predefined parent tag to replace the text and store the entire data in a new XML file. These specific tags should not contain any other nested tags. + +For example, consider an XML file named data.xml: + +**`data.xml`** + +```xml + + + + NewYork Agency + 0032 + + + Doe + 12345 + 50000 + + + Smith + 67890 + 60000 + + +``` + +In this example, you can mask the values of `agency_number` in the `agency` tag and the values of `name` and `account_number` in the `account` tag using the following command: + +```bash + `cat data.xml | pimo xml --subscriber agency=masking_agency.yml --subscriber account=masking_account.yml > maskedData.xml` +``` + +**`masking_agency.yml`** + +```yaml +version: "1" +seed: 42 + +masking: + - selector: + jsonpath: "agency_number" # this is the name of tag that will be masked + mask: + template: '{{MaskRegex "[0-9]{4}$"}}' +``` + +**`masking_account.yml`** + +```yaml +version: "1" +seed: 42 + +masking: + - selector: + jsonpath: "name" # this is the name of tag that will be masked + mask: + randomChoiceInUri: "pimo://nameFR" + - selector: + jsonpath: "account_number" # this is the name of tag that will be masked + masks: + - incremental: + start: 1 + increment: 1 + # incremental will change string to int, need to use template to restore string value in xml file + - template: "{{.account_number}}" +``` + +After executing the command with the correct configuration, here is the expected result in the file maskedData.xml: + +**`maskedData.xml`** + +```xml + + + + NewYork Agency + 2308 + + + Rolande + 1 + 50000 + + + Matéo + 2 + 60000 + + +``` + +[Return to list of masks](#possible-masks) + + ## `pimo://` scheme Pimo embed a usefule list of fake data. URIs that begin with a pimo:// sheme point to the pseudo files bellow. diff --git a/cmd/pimo/main.go b/cmd/pimo/main.go index 72a88435..6149d95a 100644 --- a/cmd/pimo/main.go +++ b/cmd/pimo/main.go @@ -67,7 +67,7 @@ var ( statsTemplate string statsDestinationEnv = os.Getenv("PIMO_STATS_URL") statsTemplateEnv = os.Getenv("PIMO_STATS_TEMPLATE") - subscriberName map[string]string + xmlSubscriberName map[string]string ) func main() { @@ -123,7 +123,7 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa // Add command for XML transformer xmlCmd := &cobra.Command{ Use: "xml", - Short: "Transform fichier XML with substitution", + Short: "Parsing and masking XML file", Run: func(cmd *cobra.Command, args []string) { initLog() if len(catchErrors) > 0 { @@ -140,12 +140,12 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa SkipLogFile: skipLogFile, CachesToDump: cachesToDump, CachesToLoad: cachesToLoad, - Callback: true, + XMLCallback: true, } // ces étapes doivent retrouver dans xixo.go pour ne pas alourdre le main.gopi parser := pimo.ParseXML(cmd.InOrStdin(), cmd.OutOrStdout()) // Map the command line balise name to fit the masking configuration - for elementName, mask := range subscriberName { + for elementName, mask := range xmlSubscriberName { pdef, err := model.LoadPipelineDefinitionFromFile(mask) if err != nil { fmt.Printf("Error when charging pipeline for %s : %v\n", elementName, err) @@ -163,11 +163,7 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa if err != nil { return nil, err } - // Update m with masked data - for key, value := range transformedData { - m[key] = value - } - return m, nil + return transformedData, nil }) } err := parser.Stream() @@ -176,7 +172,7 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa } }, } - xmlCmd.Flags().StringToStringVar(&subscriberName, "subscriber", map[string]string{}, "name of element to mask") + xmlCmd.Flags().StringToStringVar(&xmlSubscriberName, "subscriber", map[string]string{}, "name of element to mask") rootCmd.AddCommand(xmlCmd) rootCmd.AddCommand(&cobra.Command{ diff --git a/exemple_expected.xml b/exemple_expected.xml deleted file mode 100644 index 615d883f..00000000 --- a/exemple_expected.xml +++ /dev/null @@ -1 +0,0 @@ -Error when charging pipeline for account : open masking_account.yml: no such file or directory diff --git a/internal/app/pimo/pimo.go b/internal/app/pimo/pimo.go index e0e6cb4d..230f97bb 100755 --- a/internal/app/pimo/pimo.go +++ b/internal/app/pimo/pimo.go @@ -77,7 +77,7 @@ type Config struct { SkipLogFile string CachesToDump map[string]string CachesToLoad map[string]string - Callback bool + XMLCallback bool } type Context struct { @@ -108,7 +108,7 @@ func (ctx *Context) Configure(cfg Config) error { over.AddGlobalFields("context") switch { - case cfg.Callback: + case cfg.XMLCallback: over.MDC().Set("context", "callback-input") ctx.source = model.NewCallableMapSource() case cfg.EmptyInput: diff --git a/internal/app/pimo/pimo_test.go b/internal/app/pimo/pimo_test.go index 9dac75ee..925a264b 100755 --- a/internal/app/pimo/pimo_test.go +++ b/internal/app/pimo/pimo_test.go @@ -333,8 +333,8 @@ func TestExecuteMap(t *testing.T) { } ctx := pimo.NewContext(definition) cfg := pimo.Config{ - Iteration: 1, - Callback: true, + Iteration: 1, + XMLCallback: true, } err := ctx.Configure(cfg) @@ -363,8 +363,8 @@ func Test2BaliseIdentity(t *testing.T) { } ctx := pimo.NewContext(definition) cfg := pimo.Config{ - Iteration: 1, - Callback: true, + Iteration: 1, + XMLCallback: true, } err := ctx.Configure(cfg) @@ -372,6 +372,7 @@ func Test2BaliseIdentity(t *testing.T) { data := map[string]string{"name": "John"} newData1, err := ctx.ExecuteMap(data) + assert.Nil(t, err) newData2, err := ctx.ExecuteMap(data) assert.Nil(t, err) assert.NotEqual(t, newData2["name"], newData1["name"]) diff --git a/test/exemple_expected.xml b/test/exemple_expected.xml index df1bef44..d3b6739e 100644 --- a/test/exemple_expected.xml +++ b/test/exemple_expected.xml @@ -1,9 +1,9 @@ - Nantes Agency - 0032 - + Nantes Agency + 2308 + Rolande 1 From 5f96b8b4d614736be3b5f91a5afeae37139b9041 Mon Sep 17 00:00:00 2001 From: Jianchao Ma Date: Mon, 6 Nov 2023 08:50:28 +0000 Subject: [PATCH 07/16] =?UTF-8?q?feat:=20mise=20=C3=A0=20jour=20de=20xixo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/pimo/main.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- internal/app/pimo/xixo.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/pimo/main.go b/cmd/pimo/main.go index 6149d95a..42697b58 100644 --- a/cmd/pimo/main.go +++ b/cmd/pimo/main.go @@ -142,7 +142,7 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa CachesToLoad: cachesToLoad, XMLCallback: true, } - // ces étapes doivent retrouver dans xixo.go pour ne pas alourdre le main.gopi + parser := pimo.ParseXML(cmd.InOrStdin(), cmd.OutOrStdout()) // Map the command line balise name to fit the masking configuration for elementName, mask := range xmlSubscriberName { diff --git a/go.mod b/go.mod index 192e5ab2..ebef916b 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/cgi-fr/pimo go 1.20 require ( + github.com/CGI-FR/xixo v0.1.1 github.com/Masterminds/sprig/v3 v3.2.3 github.com/adrienaury/zeromdc v0.0.0-20221116212822-6a366c26ee61 github.com/capitalone/fpe v1.2.1 @@ -15,7 +16,6 @@ require ( github.com/spf13/cast v1.5.1 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 - github.com/youen/xixo v0.0.0-20230922162717-6d8c927577cb github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea gitlab.com/c0b/go-ordered-json v0.0.0-20201030195603-febf46534d5a golang.org/x/text v0.12.0 diff --git a/go.sum b/go.sum index 0c1b19bd..2d6e97b7 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/CGI-FR/xixo v0.1.1 h1:eUB3fR/iJNawd1j3tzZIZnElWJtoLzVN7HLc/+vMvB8= +github.com/CGI-FR/xixo v0.1.1/go.mod h1:8arh0a5wsw67GuWoL1t4ea8jjN+movVMZty17ahGlWA= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= @@ -103,8 +105,6 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/youen/xixo v0.0.0-20230922162717-6d8c927577cb h1:FpW08PivlO+zqpU92KxbS0jkvsiB447eAyYSfJqsn/Q= -github.com/youen/xixo v0.0.0-20230922162717-6d8c927577cb/go.mod h1:Nr//UqY2ngqBw5P+HkiYhppjS3dmqlZ9ENKDXtFrTsM= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea h1:CyhwejzVGvZ3Q2PSbQ4NRRYn+ZWv5eS1vlaEusT+bAI= github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea/go.mod h1:eNr558nEUjP8acGw8FFjTeWvSgU1stO7FAO6eknhHe4= diff --git a/internal/app/pimo/xixo.go b/internal/app/pimo/xixo.go index 37152170..ed21607e 100644 --- a/internal/app/pimo/xixo.go +++ b/internal/app/pimo/xixo.go @@ -20,7 +20,7 @@ package pimo import ( "io" - "github.com/youen/xixo/pkg/xixo" + "github.com/CGI-FR/xixo/pkg/xixo" ) func ParseXML(input io.Reader, output io.Writer) *xixo.XMLParser { From 8c683b28813b4f2f3b7ea61c296ccd80fcb7fd7a Mon Sep 17 00:00:00 2001 From: Jianchao Ma Date: Mon, 6 Nov 2023 10:13:16 +0000 Subject: [PATCH 08/16] feat: ajouter test avec attributs --- internal/app/pimo/pimo_test.go | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/internal/app/pimo/pimo_test.go b/internal/app/pimo/pimo_test.go index 925a264b..3d7ad232 100755 --- a/internal/app/pimo/pimo_test.go +++ b/internal/app/pimo/pimo_test.go @@ -377,3 +377,38 @@ func Test2BaliseIdentity(t *testing.T) { assert.Nil(t, err) assert.NotEqual(t, newData2["name"], newData1["name"]) } + +func TestExecuteMapWithAttributes(t *testing.T) { + definition := model.Definition{ + Version: "1", + Masking: []model.Masking{ + { + Selector: model.SelectorType{Jsonpath: "name"}, + Mask: model.MaskType{ + HashInURI: "pimo://nameFR", + }, + }, + { + Selector: model.SelectorType{Jsonpath: "name"}, + Mask: model.MaskType{ + HashInURI: "pimo://nameFR", + }, + }, + }, + } + ctx := pimo.NewContext(definition) + cfg := pimo.Config{ + Iteration: 1, + XMLCallback: true, + } + + err := ctx.Configure(cfg) + assert.Nil(t, err) + + data := map[string]string{"name": "John"} + + newData, err := ctx.ExecuteMap(data) + + assert.Nil(t, err) + assert.NotEqual(t, "John", newData["name"]) +} From e1043fc4fe363d00da0a94c61c97bfb5eebddd68 Mon Sep 17 00:00:00 2001 From: Jianchao Ma Date: Wed, 8 Nov 2023 12:04:18 +0000 Subject: [PATCH 09/16] feat: add venom test and update README --- README.md | 36 ++++++++++++++++++++++++++-------- go.mod | 2 +- go.sum | 4 ++-- internal/app/pimo/pimo_test.go | 36 ++++------------------------------ test/masking_attr.yml | 18 +++++++++++++++++ test/suites/parsingXML.yml | 28 ++++++++++++++++++++++++++ 6 files changed, 81 insertions(+), 43 deletions(-) create mode 100644 test/masking_attr.yml create mode 100755 test/suites/parsingXML.yml diff --git a/README.md b/README.md index 4a97a3f0..ef4b53bf 100755 --- a/README.md +++ b/README.md @@ -966,6 +966,11 @@ To use PIMO to masking data in an XML file, use in the following way : Pimo selects specific tags within a predefined parent tag to replace the text and store the entire data in a new XML file. These specific tags should not contain any other nested tags. +To mask values of attributes, follow the rules to define your choice in jsonpath in masking.yml. + +* For attributes of parent tag, we use: `@attributeName` in jsonpath. +* For attributes of child tag, we use: `childTagName@attributeName` in jsonpath. + For example, consider an XML file named data.xml: **`data.xml`** @@ -977,13 +982,13 @@ For example, consider an XML file named data.xml: NewYork Agency 0032 - - Doe + + Doe 12345 50000 - - Smith + + Smith 67890 60000 @@ -1020,6 +1025,13 @@ masking: jsonpath: "name" # this is the name of tag that will be masked mask: randomChoiceInUri: "pimo://nameFR" + - selector: + jsonpath: "@type" # this is the name of parent tag's attribute that will be masked + mask: + randomChoice: + - "classic" + - "saving" + - "securitie" - selector: jsonpath: "account_number" # this is the name of tag that will be masked masks: @@ -1028,6 +1040,14 @@ masking: increment: 1 # incremental will change string to int, need to use template to restore string value in xml file - template: "{{.account_number}}" + - selector: + jsonpath: "name@age" # this is the name of child tag's attribute that will be masked + masks: + - randomInt: + min: 18 + max: 95 + # @ is not accepted by GO, so there we need use index in template to change int into string + - template: "{{index . \"name@age\"}}" ``` After executing the command with the correct configuration, here is the expected result in the file maskedData.xml: @@ -1041,13 +1061,13 @@ After executing the command with the correct configuration, here is the expected NewYork Agency 2308 - - Rolande + + Rolande 1 50000 - - Matéo + + Matéo 2 60000 diff --git a/go.mod b/go.mod index ebef916b..afd14246 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cgi-fr/pimo go 1.20 require ( - github.com/CGI-FR/xixo v0.1.1 + github.com/CGI-FR/xixo v0.1.3 github.com/Masterminds/sprig/v3 v3.2.3 github.com/adrienaury/zeromdc v0.0.0-20221116212822-6a366c26ee61 github.com/capitalone/fpe v1.2.1 diff --git a/go.sum b/go.sum index 2d6e97b7..4d5e9461 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/CGI-FR/xixo v0.1.1 h1:eUB3fR/iJNawd1j3tzZIZnElWJtoLzVN7HLc/+vMvB8= -github.com/CGI-FR/xixo v0.1.1/go.mod h1:8arh0a5wsw67GuWoL1t4ea8jjN+movVMZty17ahGlWA= +github.com/CGI-FR/xixo v0.1.3 h1:+i2CUL4+AxdqtQ8oiIWHkPrrL/7zrlFPpb+i7tMKqxc= +github.com/CGI-FR/xixo v0.1.3/go.mod h1:8arh0a5wsw67GuWoL1t4ea8jjN+movVMZty17ahGlWA= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= diff --git a/internal/app/pimo/pimo_test.go b/internal/app/pimo/pimo_test.go index 3d7ad232..d7bba324 100755 --- a/internal/app/pimo/pimo_test.go +++ b/internal/app/pimo/pimo_test.go @@ -319,35 +319,6 @@ func LoadJsonLineFromDocument(filename string) (model.Dictionary, error) { // return jsonline.JSONToDictionary(compactLine.Bytes()) } -func TestExecuteMap(t *testing.T) { - definition := model.Definition{ - Version: "1", - Masking: []model.Masking{ - { - Selector: model.SelectorType{Jsonpath: "name"}, - Mask: model.MaskType{ - HashInURI: "pimo://nameFR", - }, - }, - }, - } - ctx := pimo.NewContext(definition) - cfg := pimo.Config{ - Iteration: 1, - XMLCallback: true, - } - - err := ctx.Configure(cfg) - assert.Nil(t, err) - - data := map[string]string{"name": "John"} - - newData, err := ctx.ExecuteMap(data) - - assert.Nil(t, err) - assert.NotEqual(t, "John", newData["name"]) -} - func Test2BaliseIdentity(t *testing.T) { definition := model.Definition{ Version: "1", @@ -389,9 +360,9 @@ func TestExecuteMapWithAttributes(t *testing.T) { }, }, { - Selector: model.SelectorType{Jsonpath: "name"}, + Selector: model.SelectorType{Jsonpath: "name@age"}, Mask: model.MaskType{ - HashInURI: "pimo://nameFR", + Regex: "([0-9]){2}", }, }, }, @@ -405,10 +376,11 @@ func TestExecuteMapWithAttributes(t *testing.T) { err := ctx.Configure(cfg) assert.Nil(t, err) - data := map[string]string{"name": "John"} + data := map[string]string{"name": "John", "name@age": "25"} newData, err := ctx.ExecuteMap(data) assert.Nil(t, err) assert.NotEqual(t, "John", newData["name"]) + assert.NotEqual(t, "25", newData["name@age"]) } diff --git a/test/masking_attr.yml b/test/masking_attr.yml new file mode 100644 index 00000000..44616c55 --- /dev/null +++ b/test/masking_attr.yml @@ -0,0 +1,18 @@ +version: "1" +seed: 42 +masking: + - selector: + jsonpath: "name" + mask: + randomChoiceInUri: "pimo://nameFR" + - selector: + jsonpath: "@name" + mask: + randomChoiceInUri: "pimo://nameFR" + - selector: + jsonpath: "name@age" + masks: + - randomInt: + min: 18 + max: 95 + - template: "{{index . \"name@age\"}}" diff --git a/test/suites/parsingXML.yml b/test/suites/parsingXML.yml new file mode 100755 index 00000000..9fbde437 --- /dev/null +++ b/test/suites/parsingXML.yml @@ -0,0 +1,28 @@ +name: parsing/masking XML features +testcases: + - name: masking XML one parent tag + steps: + - script: |- + cat > origin.xml < + + Bar + + EOF + - script: |- + cat > expected.xml < + + Rolande + + EOF + - script: |- + cat origin.xml | pimo xml --subscriber User=../masking_attr.yml > result.xml + assertions: + - result.code ShouldEqual 0 + - script: diff expected.xml result.xml + assertions: + - result.systemout ShouldBeEmpty + - script: rm -f origin.xml + - script: rm -f expected.xml + - script: rm -f result.xml From 08bb8deea292db2e1c98463e9367e9815bfe6ab1 Mon Sep 17 00:00:00 2001 From: Adrien Aury Date: Wed, 8 Nov 2023 16:54:40 +0100 Subject: [PATCH 10/16] fix: seed for xml --- cmd/pimo/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/pimo/main.go b/cmd/pimo/main.go index 42697b58..ad6503e9 100644 --- a/cmd/pimo/main.go +++ b/cmd/pimo/main.go @@ -103,7 +103,7 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa rootCmd.PersistentFlags().BoolVar(&skipFieldOnError, "skip-field-on-error", false, "remove a field if an error occurs while masking this field") rootCmd.PersistentFlags().StringVar(&skipLogFile, "skip-log-file", "", "skipped lines will be written to this log file") rootCmd.PersistentFlags().StringVarP(&catchErrors, "catch-errors", "e", "", "catch errors and write line in file, same as using skip-field-on-error + skip-log-file") - rootCmd.Flags().Int64VarP(&seedValue, "seed", "s", 0, "set seed") + rootCmd.PersistentFlags().Int64VarP(&seedValue, "seed", "s", 0, "set seed") rootCmd.PersistentFlags().StringArrayVarP(&maskingOneLiner, "mask", "m", []string{}, "one liner masking") rootCmd.PersistentFlags().StringVar(&repeatUntil, "repeat-until", "", "mask each input repeatedly until the given condition is met") rootCmd.PersistentFlags().StringVar(&repeatWhile, "repeat-while", "", "mask each input repeatedly while the given condition is met") From 349d9a7974111c72b5f79be8b87bbde54dfefa63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Youen=20P=C3=A9ron?= Date: Tue, 14 Nov 2023 10:07:45 +0000 Subject: [PATCH 11/16] feat: upgrade xixo to v0.1.5 --- go.mod | 2 +- go.sum | 4 ++-- test/exemple.xml | 13 +++++++------ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index afd14246..335d1aba 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cgi-fr/pimo go 1.20 require ( - github.com/CGI-FR/xixo v0.1.3 + github.com/CGI-FR/xixo v0.1.5 github.com/Masterminds/sprig/v3 v3.2.3 github.com/adrienaury/zeromdc v0.0.0-20221116212822-6a366c26ee61 github.com/capitalone/fpe v1.2.1 diff --git a/go.sum b/go.sum index 4d5e9461..e61bcc46 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/CGI-FR/xixo v0.1.3 h1:+i2CUL4+AxdqtQ8oiIWHkPrrL/7zrlFPpb+i7tMKqxc= -github.com/CGI-FR/xixo v0.1.3/go.mod h1:8arh0a5wsw67GuWoL1t4ea8jjN+movVMZty17ahGlWA= +github.com/CGI-FR/xixo v0.1.5 h1:q+5pdCeAYKKF9E/Ac+9rXnoul1TzbiAXDwg5XEfm2uM= +github.com/CGI-FR/xixo v0.1.5/go.mod h1:8arh0a5wsw67GuWoL1t4ea8jjN+movVMZty17ahGlWA= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= diff --git a/test/exemple.xml b/test/exemple.xml index aa3187b4..e3ad5df0 100644 --- a/test/exemple.xml +++ b/test/exemple.xml @@ -16,10 +16,11 @@ 60000 12000 - - Hello world - 00000 - 60000 - 12000 - + + Hello world + + 00000 + 60000 + 12000 + From 8ab87b1d8ee08e895b7ae8833021b46815b3141b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Youen=20P=C3=A9ron?= Date: Thu, 16 Nov 2023 21:51:27 +0000 Subject: [PATCH 12/16] fix: upgrade xixo --- go.mod | 3 +-- go.sum | 7 ++----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 335d1aba..01352659 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cgi-fr/pimo go 1.20 require ( - github.com/CGI-FR/xixo v0.1.5 + github.com/CGI-FR/xixo v0.1.6 github.com/Masterminds/sprig/v3 v3.2.3 github.com/adrienaury/zeromdc v0.0.0-20221116212822-6a366c26ee61 github.com/capitalone/fpe v1.2.1 @@ -41,7 +41,6 @@ require ( github.com/shopspring/decimal v1.2.0 // indirect github.com/smartystreets/goconvey v1.6.4 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/tamerh/xpath v1.0.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect golang.org/x/crypto v0.11.0 // indirect diff --git a/go.sum b/go.sum index e61bcc46..65e2a849 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/CGI-FR/xixo v0.1.5 h1:q+5pdCeAYKKF9E/Ac+9rXnoul1TzbiAXDwg5XEfm2uM= -github.com/CGI-FR/xixo v0.1.5/go.mod h1:8arh0a5wsw67GuWoL1t4ea8jjN+movVMZty17ahGlWA= +github.com/CGI-FR/xixo v0.1.6 h1:C3BPzLmUebjXsQqaP8A6IBwtlqpRX6Pq9xd3PQp6DCg= +github.com/CGI-FR/xixo v0.1.6/go.mod h1:Q7Xf6CHqoU6hyRwPtvrUu4wCspfFYxIWZoYXTYXvtI8= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= @@ -8,7 +8,6 @@ github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/adrienaury/zeromdc v0.0.0-20221116212822-6a366c26ee61 h1:GXmgCyG0oDr3BFL+NYcwKEP2H3L9DuBo66ZR++zNasA= github.com/adrienaury/zeromdc v0.0.0-20221116212822-6a366c26ee61/go.mod h1:5UlMlw0MRjEAms20gDadR5GrN2wEp9XEXTBiMp/XI4E= -github.com/antchfx/xpath v1.2.4 h1:dW1HB/JxKvGtJ9WyVGJ0sIoEcqftV3SqIstujI+B9XY= github.com/capitalone/fpe v1.2.1 h1:/r81KhhTkfmxjjr2HKr+WYTLrMjPnn0gtK/L8gKNfts= github.com/capitalone/fpe v1.2.1/go.mod h1:hI6YzL2v2WkosaevH24sYHyyDAzacfqkpaOYc/0Qn7g= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -98,8 +97,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tamerh/xpath v1.0.0 h1:NccMES/Ej8slPCFDff73Kf6V1xu9hdbuKf2RyDsxf5Q= -github.com/tamerh/xpath v1.0.0/go.mod h1:t0wnh72FQlOVEO20f2Dl3EoVxso9GnLREh1WTpvNmJQ= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= From bef94cfab4ff3866225bb5339ca7c3e286833431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Youen=20P=C3=A9ron?= Date: Sun, 19 Nov 2023 12:26:01 +0000 Subject: [PATCH 13/16] fix: conflict with --seed shortcut and --secure shortcut in pimo play --- cmd/pimo/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/pimo/main.go b/cmd/pimo/main.go index ad6503e9..42697b58 100644 --- a/cmd/pimo/main.go +++ b/cmd/pimo/main.go @@ -103,7 +103,7 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa rootCmd.PersistentFlags().BoolVar(&skipFieldOnError, "skip-field-on-error", false, "remove a field if an error occurs while masking this field") rootCmd.PersistentFlags().StringVar(&skipLogFile, "skip-log-file", "", "skipped lines will be written to this log file") rootCmd.PersistentFlags().StringVarP(&catchErrors, "catch-errors", "e", "", "catch errors and write line in file, same as using skip-field-on-error + skip-log-file") - rootCmd.PersistentFlags().Int64VarP(&seedValue, "seed", "s", 0, "set seed") + rootCmd.Flags().Int64VarP(&seedValue, "seed", "s", 0, "set seed") rootCmd.PersistentFlags().StringArrayVarP(&maskingOneLiner, "mask", "m", []string{}, "one liner masking") rootCmd.PersistentFlags().StringVar(&repeatUntil, "repeat-until", "", "mask each input repeatedly until the given condition is met") rootCmd.PersistentFlags().StringVar(&repeatWhile, "repeat-while", "", "mask each input repeatedly while the given condition is met") From 206956299eac1f4c7b8092b0e82e3e6f225721ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Youen=20P=C3=A9ron?= Date: Mon, 20 Nov 2023 08:48:35 +0000 Subject: [PATCH 14/16] feat: support seed from cli for xml parsing --- cmd/pimo/main.go | 6 ++++++ test/suites/parsingXML.yml | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/cmd/pimo/main.go b/cmd/pimo/main.go index 42697b58..2fab0ab1 100644 --- a/cmd/pimo/main.go +++ b/cmd/pimo/main.go @@ -151,6 +151,11 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa fmt.Printf("Error when charging pipeline for %s : %v\n", elementName, err) return } + + if cmd.Flags().Changed("seed") { + (&pdef).SetSeed(seedValue) + } + ctx := pimo.NewContext(pdef) if err := ctx.Configure(config); err != nil { log.Err(err).Msg("Cannot configure pipeline") @@ -173,6 +178,7 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa }, } xmlCmd.Flags().StringToStringVar(&xmlSubscriberName, "subscriber", map[string]string{}, "name of element to mask") + xmlCmd.Flags().Int64VarP(&seedValue, "seed", "s", 0, "set seed") rootCmd.AddCommand(xmlCmd) rootCmd.AddCommand(&cobra.Command{ diff --git a/test/suites/parsingXML.yml b/test/suites/parsingXML.yml index 9fbde437..1c4448b8 100755 --- a/test/suites/parsingXML.yml +++ b/test/suites/parsingXML.yml @@ -26,3 +26,30 @@ testcases: - script: rm -f origin.xml - script: rm -f expected.xml - script: rm -f result.xml + + - name: masking XML one parent tag with a given seed from cli + steps: + - script: |- + cat > origin.xml < + + Bar + + EOF + - script: |- + cat > expected.xml < + + Zacharie + + EOF + - script: |- + cat origin.xml | pimo xml --seed 41 --subscriber User=../masking_attr.yml > result.xml + assertions: + - result.code ShouldEqual 0 + - script: diff expected.xml result.xml + assertions: + - result.systemout ShouldBeEmpty + - script: rm -f origin.xml + - script: rm -f expected.xml + - script: rm -f result.xml From 8be9df3f0c1f78222d8d340980637645512d3994 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Youen=20P=C3=A9ron?= Date: Mon, 20 Nov 2023 10:42:30 +0000 Subject: [PATCH 15/16] chore: restore go.mod in node_modules --- web/play/node_modules/go.mod | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 web/play/node_modules/go.mod diff --git a/web/play/node_modules/go.mod b/web/play/node_modules/go.mod new file mode 100644 index 00000000..e69de29b From f1cf6d4fe50f77513906c76fe58b5bad68e3cfff Mon Sep 17 00:00:00 2001 From: "jianchao.ma" Date: Mon, 20 Nov 2023 14:48:09 +0000 Subject: [PATCH 16/16] feat: update xixo version 1.7 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 01352659..64d81d36 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cgi-fr/pimo go 1.20 require ( - github.com/CGI-FR/xixo v0.1.6 + github.com/CGI-FR/xixo v0.1.7 github.com/Masterminds/sprig/v3 v3.2.3 github.com/adrienaury/zeromdc v0.0.0-20221116212822-6a366c26ee61 github.com/capitalone/fpe v1.2.1 diff --git a/go.sum b/go.sum index 65e2a849..a48f305c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/CGI-FR/xixo v0.1.6 h1:C3BPzLmUebjXsQqaP8A6IBwtlqpRX6Pq9xd3PQp6DCg= -github.com/CGI-FR/xixo v0.1.6/go.mod h1:Q7Xf6CHqoU6hyRwPtvrUu4wCspfFYxIWZoYXTYXvtI8= +github.com/CGI-FR/xixo v0.1.7 h1:Qg6UqO6jiKCMAJeRbw3PFC48OBDgGpm5Em7oS5yXzDs= +github.com/CGI-FR/xixo v0.1.7/go.mod h1:Q7Xf6CHqoU6hyRwPtvrUu4wCspfFYxIWZoYXTYXvtI8= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g=