Modify CSV file generators (#292)

Author: dcollie2

app/services/csv_generator/base.rb

@@ -1,6 +1,6 @@
 class CsvGenerator::Base
   def perform
-    CSV.generate do |csv|
+    CSV.generate(row_sep: "\n") do |csv|
       csv << headers
       scope.each do |row|
         csv << row

app/services/csv_generator/files.rb

@@ -9,14 +9,15 @@ def initialize(language, **args)
   attr_reader :language, :args
 
   def headers
-    %w[TopicID FileName FileType FileSize]
+    %w[FileID TopicID FileName FileType FileSize]
   end
 
   def scope
     language.topics.active # try joining documents
       .flat_map do |topic|
         topic.documents.map do |doc|
           [
+            doc.id,
             topic.id,
             doc.filename,
             doc.content_type,

app/services/csv_generator/topic_authors.rb

@@ -0,0 +1,18 @@
+class CsvGenerator::TopicAuthors < CsvGenerator::Base
+  def initialize(language, **args)
+    @language = language
+    @args = args
+  end
+
+  private
+
+  attr_reader :language, :args
+
+  def headers
+    %w[TopicID AuthorID]
+  end
+
+  def scope
+    language.topics.active.map { |topic| [ topic.id, 0 ] }
+  end
+end

app/services/csv_generator/topics.rb

@@ -9,7 +9,7 @@ def initialize(language, **args)
   attr_reader :language, :args
 
   def headers
-    %w[TopicID TopicName TopicVolume TopicYear TopicMonth ContentProvider]
+    %w[TopicID TopicName TopicVolume TopicIssue TopicYear TopicMonth ContentProvider]
   end
 
   def scope
@@ -20,6 +20,7 @@ def scope
           topic.id,
           topic.title,
           topic.published_at.year,
+          topic.published_at.month,
           topic.published_at.year,
           topic.published_at.month,
           topic.provider.name,

app/services/language_content_processor.rb

@@ -65,6 +65,11 @@ def language_files
         name: "#{language.file_storage_prefix}TopicTag.csv",
         path: "#{language.file_storage_prefix}CMES-v2/assets/csv",
       ),
+        topic_authors: FileToUpload.new(
+        content: ->(language) { CsvGenerator::TopicAuthors.new(language).perform },
+        name: "#{language.file_storage_prefix}TopicAuthor.csv",
+        path: "#{language.file_storage_prefix}CMES-v2/assets/csv",
+      ),
     }
   end
 

spec/services/csv_generator/files_spec.rb

@@ -4,7 +4,7 @@
   subject { described_class.new(language) }
 
   let(:language) { create(:language) }
-  let(:header) { "TopicID,FileName,FileType,FileSize\n" }
+  let(:header) { "FileID,TopicID,FileName,FileType,FileSize\n" }
 
   it "generates empty csv" do
     expect(subject.perform).to eq(header)
@@ -15,7 +15,7 @@
     let(:data) do
       header.tap do |csv|
         topic.documents.each do |document|
-          csv << "#{topic.id},#{document.filename},#{document.content_type},#{document.byte_size}\n"
+          csv << "#{document.id},#{topic.id},#{document.filename},#{document.content_type},#{document.byte_size}\n"
         end
       end
     end

spec/services/csv_generator/topic_authors_spec.rb

@@ -0,0 +1,42 @@
+require "rails_helper"
+
+RSpec.describe CsvGenerator::TopicAuthors do
+  subject { described_class.new(language) }
+
+  let(:language) { create(:language) }
+  let(:header) { "TopicID,AuthorID\n" }
+
+  it "generates empty csv" do
+    expect(subject.perform).to eq(header)
+  end
+
+  context "when topics exist" do
+    let!(:topic) { create(:topic, language:) }
+    let(:data) do
+      header.tap do |csv|
+        csv << "#{topic.id},0\n"
+      end
+    end
+
+    it "generates csv with topics info" do
+      expect(subject.perform).to eq(data)
+    end
+  end
+
+  context "when topic exists but archived" do
+    let!(:topic) { create(:topic, :archived, language:) }
+
+    it "generates empty csv" do
+      expect(subject.perform).to eq(header)
+    end
+  end
+
+  context "when topic does not belong to language" do
+    let(:other_language) { create(:language) }
+    let!(:topic) { create(:topic, language: other_language) }
+
+    it "generates empty csv" do
+      expect(subject.perform).to eq(header)
+    end
+  end
+end

spec/services/csv_generator/topics_spec.rb

@@ -4,7 +4,7 @@
   subject { described_class.new(language) }
 
   let(:language) { create(:language) }
-  let(:header) { "TopicID,TopicName,TopicVolume,TopicYear,TopicMonth,ContentProvider\n" }
+  let(:header) { "TopicID,TopicName,TopicVolume,TopicIssue,TopicYear,TopicMonth,ContentProvider\n" }
 
   it "generates empty csv" do
     expect(subject.perform).to eq(header)
@@ -14,7 +14,7 @@
     let!(:topic) { create(:topic, language:) }
     let(:data) do
       header.tap do |csv|
-        csv << "#{topic.id},#{topic.title},#{topic.published_at.year},#{topic.published_at.year},#{topic.published_at.month},#{topic.provider.name}\n"
+        csv << "#{topic.id},#{topic.title},#{topic.published_at.year},#{topic.published_at.month},#{topic.published_at.year},#{topic.published_at.month},#{topic.provider.name}\n"
       end
     end
 

spec/services/language_content_processor_spec.rb

@@ -15,7 +15,7 @@
   end
 
   it "processes content for every language" do
-    files_number = language.providers.size + 8 # 2 xml files for all provides, 2 text files for tags, 4 csv files
+    files_number = language.providers.size + 9 # 2 xml files for all provides, 2 text files for tags, 5 csv files
     subject.perform
 
     expect(FileUploadJob).to have_received(:perform_later).exactly(files_number).times
@@ -28,6 +28,7 @@
     expect(FileUploadJob).to have_received(:perform_later).with(language.id, :topics)
     expect(FileUploadJob).to have_received(:perform_later).with(language.id, :tag_details)
     expect(FileUploadJob).to have_received(:perform_later).with(language.id, :topic_tags)
+    expect(FileUploadJob).to have_received(:perform_later).with(language.id, :topic_authors)
     expect(FileUploadJob).to have_received(:perform_later).with(language.id, nil, provider.id)
   end
 end