[import] Warn before CSV update creates unmatched entities

src/components/modals/ImportRenderModal.vue

@@ -131,6 +131,19 @@
       </table>
     </div>
 
+    <div class="new-entities" v-if="updateData && newEntityNames.length > 0">
+      <p class="new-entities-text">
+        {{ $t('main.csv.new_entities_to_create', newEntityNames.length) }}
+      </p>
+      <p class="new-entities-names">
+        {{ newEntityNames.join(', ') }}
+      </p>
+      <checkbox
+        :label="$t('main.csv.new_entities_confirmation', newEntityNames.length)"
+        v-model="newEntitiesConfirmed"
+      />
+    </div>
+
     <div class="render-footer">
       <button-simple
         :text="$t('main.csv.preview_reupload')"
@@ -139,7 +152,7 @@
       <modal-footer
         :error-text="errorText"
         :is-loading="isLoading"
-        :is-disabled="formData === undefined"
+        :is-disabled="isConfirmDisabled"
         :is-error="isError"
         @confirm="$emit('confirm', parsedCsv, updateData)"
         @cancel="$emit('cancel')"
@@ -149,11 +162,13 @@
 </template>
 
 <script setup>
-import { computed, ref } from 'vue'
+import { computed, ref, watch } from 'vue'
 import { useI18n } from 'vue-i18n'
 import { useRoute } from 'vue-router'
 import { useStore } from 'vuex'
 
+import csv from '@/lib/csv'
+
 import BaseModal from '@/components/modals/BaseModal.vue'
 import ModalFooter from '@/components/modals/ModalFooter.vue'
 import ButtonSimple from '@/components/widgets/ButtonSimple.vue'
@@ -179,7 +194,7 @@ const props = defineProps({
 defineEmits(['cancel', 'confirm', 'reupload'])
 
 const duplicates = ref([])
-const formData = ref(null)
+const newEntitiesConfirmed = ref(false)
 const updateData = ref(false)
 
 const assetMetadataDescriptors = computed(
@@ -243,6 +258,22 @@ const indexMatchers = computed(() =>
   props.dataMatchers.map(item => props.parsedCsv[0].indexOf(item))
 )
 
+const newEntityNames = computed(() => {
+  if (props.parsedCsv.length === 0) return []
+  return csv.getNewEntityNames(
+    props.parsedCsv,
+    indexMatchers.value,
+    props.database
+  )
+})
+
+const isConfirmDisabled = computed(
+  () =>
+    updateData.value &&
+    newEntityNames.value.length > 0 &&
+    !newEntitiesConfirmed.value
+)
+
 const errorText = computed(() => {
   let text = t('main.csv.error_upload')
   if (props.importError?.status === 400) {
@@ -266,13 +297,20 @@ const isDuplicated = index =>
   duplicates.value.includes(columnSelect.value[index])
 
 const existingData = index => {
-  const csv = props.parsedCsv[index + 1]
+  const line = props.parsedCsv[index + 1]
   let itemName = ''
   indexMatchers.value.forEach(col => {
-    itemName += csv[col]
+    itemName += line[col]
   })
   return props.database[itemName]
 }
+
+watch(
+  () => props.active,
+  () => {
+    newEntitiesConfirmed.value = false
+  }
+)
 </script>
 
 <style lang="scss" scoped>
@@ -359,6 +397,26 @@ const existingData = index => {
   border-bottom: 1px solid $light-grey-light;
 }
 
+.new-entities {
+  border: 1px solid $orange-carrot;
+  border-radius: 4px;
+  margin-top: 1em;
+  padding: 1em;
+
+  .new-entities-text {
+    color: var(--text);
+    font-weight: bold;
+    margin-bottom: 0.5em;
+  }
+
+  .new-entities-names {
+    color: var(--text);
+    margin-bottom: 1em;
+    max-height: 100px;
+    overflow: auto;
+  }
+}
+
 .render-footer {
   display: flex;
   align-items: center;

src/components/pages/Edits.vue

@@ -460,8 +460,13 @@ export default {
     },
 
     filteredEdits() {
+      // Build the lookup from the full edit cache, not the filtered display
+      // list, so the import creation check sees every edit.
+      // The cache Map is not reactive: depend on displayedEdits (updated
+      // by the same mutations) to invalidate this computed.
+      this.displayedEdits // eslint-disable-line no-unused-expressions
       const edits = {}
-      this.displayedEdits.forEach(edit => {
+      this.editMap.forEach(edit => {
         let editKey = ''
         if (
           this.isTVShow &&

src/components/pages/Shots.vue

@@ -560,15 +560,18 @@ export default {
     },
 
     filteredShots() {
+      // Build the lookup from the full shot cache, not the filtered display
+      // list, so the import creation check sees every shot.
+      // The cache Map is not reactive: depend on displayedShots (updated
+      // by the same mutations) to invalidate this computed.
+      this.displayedShots // eslint-disable-line no-unused-expressions
       const shots = {}
-      this.displayedShotsBySequence.forEach(sequence => {
-        sequence.forEach(item => {
-          let shotKey = `${item.sequence_name}${item.name}`
-          if (this.isTVShow) {
-            shotKey = item.episode_name + shotKey
-          }
-          shots[shotKey] = true
-        })
+      this.shotMap.forEach(item => {
+        let shotKey = `${item.sequence_name}${item.name}`
+        if (this.isTVShow) {
+          shotKey = item.episode_name + shotKey
+        }
+        shots[shotKey] = true
       })
       return shots
     },

src/lib/csv.js

@@ -192,6 +192,27 @@ const csv = {
     return stringHelpers.slugify(nameData.join('_'))
   },
 
+  /*
+   * Return the display names of the parsed CSV lines that don't match any
+   * existing entity in the given lookup (those lines will be created by the
+   * import instead of updating an existing entity). `indexMatchers` lists
+   * the column indexes used to build the lookup keys.
+   */
+  getNewEntityNames(parsedCsv, indexMatchers, database) {
+    const names = new Map()
+    parsedCsv
+      .slice(1)
+      .filter(line => line.length > 1)
+      .forEach(line => {
+        const values = indexMatchers.map(index => line[index] || '')
+        const key = values.join('')
+        if (key.length > 0 && !database[key] && !names.has(key)) {
+          names.set(key, values.filter(value => value.length > 0).join(' / '))
+        }
+      })
+    return [...names.values()]
+  },
+
   turnEntriesToCsvString(entries, config = {}) {
     return Papa.unparse(entries, {
       delimiter: ';',

src/locales/en.js

@@ -945,6 +945,8 @@ export default {
       legend_missing_optional: 'Optional column that was not found',
       legend_disabled: 'Line that will not be updated or created',
       legend_overwrite: 'Line that will be updated',
+      new_entities_confirmation: 'I confirm that I want to create this new entry | I confirm that I want to create these {count} new entries',
+      new_entities_to_create: '{count} line does not match any existing entry and will be created as a new entry: | {count} lines do not match any existing entries and will be created as new entries:',
       paste: 'Paste',
       paste_code: 'Please paste your CSV data here:',
       preview: 'Preview',

tests/unit/lib/csv.spec.js

@@ -33,4 +33,75 @@ describe('lib/csv', () => {
       expect(result).toBe('"a";"b"')
     })
   })
+
+  describe('getNewEntityNames', () => {
+    const parsedCsv = [
+      ['Sequence', 'Name', 'Description'],
+      ['SEQ01', 'SH001', 'A shot'],
+      ['SEQ01', 'SH002', 'Another shot'],
+      ['SEQ02', 'SH001', 'Yet another shot']
+    ]
+    const indexMatchers = [0, 1]
+
+    test('returns lines that do not match any existing entity', () => {
+      const database = { SEQ01SH001: true }
+      expect(csv.getNewEntityNames(parsedCsv, indexMatchers, database)).toEqual(
+        ['SEQ01 / SH002', 'SEQ02 / SH001']
+      )
+    })
+
+    test('returns no name when every line matches', () => {
+      const database = {
+        SEQ01SH001: true,
+        SEQ01SH002: true,
+        SEQ02SH001: true
+      }
+      expect(csv.getNewEntityNames(parsedCsv, indexMatchers, database)).toEqual(
+        []
+      )
+    })
+
+    test('detects names altered by the spreadsheet (issue #771)', () => {
+      const database = { SEQ01SH001: true, SEQ01SH002: true }
+      const alteredCsv = [
+        ['Sequence', 'Name'],
+        ['SEQ01', 'SH1'],
+        ['SEQ01', 'SH002']
+      ]
+      expect(
+        csv.getNewEntityNames(alteredCsv, indexMatchers, database)
+      ).toEqual(['SEQ01 / SH1'])
+    })
+
+    test('deduplicates lines sharing the same matcher key', () => {
+      const duplicatedCsv = [
+        ['Sequence', 'Name'],
+        ['SEQ01', 'SH001'],
+        ['SEQ01', 'SH001']
+      ]
+      expect(csv.getNewEntityNames(duplicatedCsv, indexMatchers, {})).toEqual([
+        'SEQ01 / SH001'
+      ])
+    })
+
+    test('ignores empty and single-cell lines', () => {
+      const sparseCsv = [
+        ['Sequence', 'Name'],
+        [''],
+        ['SEQ01', 'SH001'],
+        ['', '']
+      ]
+      expect(csv.getNewEntityNames(sparseCsv, indexMatchers, {})).toEqual([
+        'SEQ01 / SH001'
+      ])
+    })
+
+    test('handles matcher columns missing from the line', () => {
+      const shortCsv = [
+        ['Sequence', 'Name', 'Description'],
+        ['SEQ01', 'SH001']
+      ]
+      expect(csv.getNewEntityNames(shortCsv, [0, 5], {})).toEqual(['SEQ01'])
+    })
+  })
 })