feat: support comment in csv (#304)

examples/basic/basic.int.delete.v3.yaml

@@ -25,6 +25,7 @@ sources:
   - path: ./person.csv
     csv:
       delimiter: "|"
+      comment: "#"
     tags:
     - name: Person
       mode: DELETE

examples/basic/basic.int.insert.v3.yaml

@@ -35,6 +35,7 @@ sources:
   - path: ./person.csv
     csv:
       delimiter: "|"
+      comment: "#"
     tags:
     - name: Person
       id:

examples/basic/basic.int.update.v3.yaml

@@ -25,6 +25,7 @@ sources:
   - path: ./person.csv
     csv:
       delimiter: "|"
+      comment: "#"
     tags:
     - name: Person
       mode: UPDATE

examples/basic/basic.string.delete.v3.yaml

@@ -25,6 +25,7 @@ sources:
   - path: ./person.csv
     csv:
       delimiter: "|"
+      comment: "#"
     tags:
     - name: Person
       mode: DELETE

examples/basic/basic.string.insert.v3.yaml

@@ -35,6 +35,7 @@ sources:
   - path: ./person.csv
     csv:
       delimiter: "|"
+      comment: "#"
     tags:
     - name: Person
       id:

examples/basic/basic.string.update.v3.yaml

@@ -25,6 +25,7 @@ sources:
   - path: ./person.csv
     csv:
       delimiter: "|"
+      comment: "#"
     tags:
     - name: Person
       mode: UPDATE

examples/basic/person.csv

@@ -1,4 +1,7 @@
+# I'm comment
+# I'm comment
 933|Mahinda|Perera|male|1989-12-03|2010-02-14T15:32:10|119.235.7.103|Firefox
+# I'm comment
 1129|Carmen|Lepland|female|1984-02-18|2010-01-28T06:39:58|195.20.151.175|Internet Explorer
 2199023256684|A.|Rao|female|1985-08-02|2010-04-23T22:52:26|49.202.188.25|Firefox
 4398046512167|Gustavo|Arbelaez|male|1986-11-02|2010-06-16T20:53:47|190.96.189.165|Chrome

integration-testing/testdata/basic/basic.int.delete.v3.yaml

@@ -25,6 +25,7 @@ sources:
   - path: ./person.csv
     csv:
       delimiter: "|"
+      comment: "#"
     tags:
     - name: Person
       mode: DELETE

integration-testing/testdata/basic/basic.int.insert.v3.yaml

@@ -35,6 +35,7 @@ sources:
   - path: ./person.csv
     csv:
       delimiter: "|"
+      comment: "#"
     tags:
     - name: Person
       id:

integration-testing/testdata/basic/basic.int.update.v3.yaml

@@ -25,6 +25,7 @@ sources:
   - path: ./person.csv
     csv:
       delimiter: "|"
+      comment: "#"
     tags:
     - name: Person
       mode: UPDATE

integration-testing/testdata/basic/basic.string.delete.v3.yaml

@@ -25,6 +25,7 @@ sources:
   - path: ./person.csv
     csv:
       delimiter: "|"
+      comment: "#"
     tags:
     - name: Person
       mode: DELETE

integration-testing/testdata/basic/basic.string.insert.v3.yaml

@@ -35,6 +35,7 @@ sources:
   - path: ./person.csv
     csv:
       delimiter: "|"
+      comment: "#"
     tags:
     - name: Person
       id:

integration-testing/testdata/basic/basic.string.update.v3.yaml

@@ -25,6 +25,7 @@ sources:
   - path: ./person.csv
     csv:
       delimiter: "|"
+      comment: "#"
     tags:
     - name: Person
       mode: UPDATE

integration-testing/testdata/basic/person.csv

@@ -1,4 +1,7 @@
+# I'm comment
+# I'm comment
 933|Mahinda|Perera|male|1989-12-03|2010-02-14T15:32:10|119.235.7.103|Firefox
+# I'm comment
 1129|Carmen|Lepland|female|1984-02-18|2010-01-28T06:39:58|195.20.151.175|Internet Explorer
 2199023256684|A.|Rao|female|1985-08-02|2010-04-23T22:52:26|49.202.188.25|Firefox
 4398046512167|Gustavo|Arbelaez|male|1986-11-02|2010-06-16T20:53:47|190.96.189.165|Chrome

pkg/reader/csv.go

@@ -40,6 +40,9 @@ func NewCSVReader(s source.Source) RecordReader {
 		if chars := []rune(c.CSV.Delimiter); len(chars) > 0 {
 			cr.Comma = chars[0]
 		}
+		if chars := []rune(c.CSV.Comment); len(chars) > 0 {
+			cr.Comment = chars[0]
+		}
 		cr.LazyQuotes = c.CSV.LazyQuotes
 
 		h.withHeader = c.CSV.WithHeader

pkg/reader/csv_test.go

@@ -81,6 +81,7 @@ var _ = Describe("csvReader", func() {
 				},
 				CSV: &source.CSVConfig{
 					Delimiter: "|",
+					Comment:   "#",
 				},
 			})
 			Expect(err).NotTo(HaveOccurred())
@@ -103,32 +104,32 @@ var _ = Describe("csvReader", func() {
 			r := NewCSVReader(s)
 			nBytes, err = r.Size()
 			Expect(err).NotTo(HaveOccurred())
-			Expect(nBytes).To(Equal(int64(33)))
+			Expect(nBytes).To(Equal(int64(117)))
 
 			n, record, err = r.Read()
 			Expect(err).NotTo(HaveOccurred())
-			Expect(n).To(Equal(6))
+			Expect(n).To(Equal(20))
 			Expect(record).To(Equal(spec.Record{"1", "2", "3"}))
 
 			n, record, err = r.Read()
 			Expect(err).NotTo(HaveOccurred())
-			Expect(n).To(Equal(7))
+			Expect(n).To(Equal(35))
 			Expect(record).To(Equal(spec.Record{"4", " 5", "6"}))
 
 			n, record, err = r.Read()
 			Expect(err).NotTo(HaveOccurred())
-			Expect(n).To(Equal(8))
+			Expect(n).To(Equal(22))
 			Expect(record).To(Equal(spec.Record{" 7", "8", " 9"}))
 
 			n, record, err = r.Read()
 			Expect(err).NotTo(HaveOccurred())
-			Expect(n).To(Equal(12))
+			Expect(n).To(Equal(26))
 			Expect(record).To(Equal(spec.Record{"10", " 11 ", " 12"}))
 
 			n, record, err = r.Read()
 			Expect(err).To(HaveOccurred())
 			Expect(stderrors.Is(err, io.EOF)).To(BeTrue())
-			Expect(n).To(Equal(0))
+			Expect(n).To(Equal(14))
 			Expect(record).To(BeEmpty())
 		})
 	})

pkg/reader/testdata/local_delimiter.csv

@@ -1,4 +1,10 @@
+# I'm comment
 1|2|3
+# I'm comment
+# I'm comment
 4| 5|6
+# I'm comment
  7|8| 9
+# I'm comment
 10| 11 | 12
+# I'm comment

pkg/source/config.go

@@ -15,6 +15,7 @@ type (
 
 	CSVConfig struct {
 		Delimiter  string `yaml:"delimiter,omitempty"`
+		Comment    string `yaml:"comment,omitempty"`
 		WithHeader bool   `yaml:"withHeader,omitempty"`
 		LazyQuotes bool   `yaml:"lazyQuotes,omitempty"`
 	}