File

lib/common/xml-data-bridge/xml-data-bridge-import.service.ts

Description

XmlDataBridgeImportService is responsible for importing data through XML.

Extends

XdbImportService

Index

Methods
Accessors

Constructor

constructor(dataSource: DataSource, logger: Logger, mediaService: MediaManager, filesService: FileManager, cacheService: CacheService, explorerService: ExplorerService)
Parameters :
Name Type Optional
dataSource DataSource No
logger Logger No
mediaService MediaManager No
filesService FileManager No
cacheService CacheService No
explorerService ExplorerService No

Methods

Private Async createEntityFromRowData
createEntityFromRowData(repository: Repository, rowData: literal type)

Create a new entity with properties from the provided row data.

Parameters :
Name Type Optional Description
repository Repository<any> No
  • A TypeORM Repository instance for the entity.
rowData literal type No
  • An object containing row data.
Returns : unknown

The created entity.

Private findReadOperator
findReadOperator(input: string)

Finds a read operator in a string.

Parameters :
Name Type Optional Description
input string No
  • The input string.
Returns : any

The found read operator.

Private Async getLocalizedStrings
getLocalizedStrings(row: literal type)

Build localized strings object from xml-row

Parameters :
Name Type Optional Description
row literal type No
  • A xml-row object containing the name property.

A promise that resolves to an array of LocalizedStringEntity objects.

Private getRowDataWhereConditions
getRowDataWhereConditions(rowData: XdbRowData | FileRow)

Get the WHERE conditions for a row data object.

Parameters :
Name Type Optional Description
rowData XdbRowData | FileRow No
  • An XdbRowData object containing data for a row.
Returns : object

An object containing WHERE conditions.

Private getUniqueKeyFields
getUniqueKeyFields(repository: Repository, rowData: literal type)

Get the unique key fields for an entity based on the provided row data.

Parameters :
Name Type Optional Description
repository Repository<any> No
  • A TypeORM Repository instance for the entity.
rowData literal type No
  • An object containing row data.
Returns : object

An object containing unique key fields.

Private hasPushMode
hasPushMode(item: XdbAction)

Checks whether an entity has a mode property with push value

Parameters :
Name Type Optional Description
item XdbAction No
  • xml action-node
Returns : boolean
Async importFromFile
importFromFile(fileData: Buffer)
Inherited from XdbImportService

Import XML data from Zip-archive.

Parameters :
Name Type Optional Description
fileData Buffer No
  • The XdbObject containing the XML data.
Returns : unknown

A promise that resolves to a boolean indicating whether the import was successful.

Async importXml
importXml(xml: XdbObject)
Inherited from XdbImportService

Import XML data from an XdbObject.

Parameters :
Name Type Optional Description
xml XdbObject No
  • The XdbObject containing the XML data.
Returns : Promise<boolean>

A promise that resolves to a boolean indicating whether the import was successful.

Private isColumnUnique
isColumnUnique(metadata: EntityMetadata, column: ColumnMetadata)

Determine if a column is unique.

Parameters :
Name Type Optional Description
metadata EntityMetadata No
  • EntityMetadata for the entity.
column ColumnMetadata No
  • ColumnMetadata for the column.
Returns : boolean

A boolean indicating whether the column is unique.

Private logRemovedEntity
logRemovedEntity(repository: Repository, whereConditions: object)

Log that an entity has been removed.

Parameters :
Name Type Optional Description
repository Repository<any> No
  • A TypeORM Repository instance for the entity.
whereConditions object No
  • An object containing WHERE conditions.
Returns : void
Private logSavedEntity
logSavedEntity(repository: Repository, entity: any, uniqueKeyFields: object, existingEntity: any)

Log that an entity has been saved (created or updated).

Parameters :
Name Type Optional Description
repository Repository<any> No
  • A TypeORM Repository instance for the entity.
entity any No
  • The saved entity.
uniqueKeyFields object No
  • An object containing unique key fields.
existingEntity any No
  • An existing entity, if any.
Returns : void
Private Async processFileNodes
processFileNodes(item: XdbAction)

Process "file" nodes by creating or updating a file.

Parameters :
Name Type Optional Description
item XdbAction No
  • An XdbActions object containing rows of file data.
Returns : any

A promise that resolves when all file nodes are processed.

Private Async processIncludeNodes
processIncludeNodes(item: XdbAction)

Process "file" nodes by creating or updating a file.

Parameters :
Name Type Optional Description
item XdbAction No
  • An XdbActions object containing rows of file data.
Returns : any

A promise that resolves when all file nodes are processed.

Private Async processInsertUpdateNodes
processInsertUpdateNodes(item: XdbAction)

Process "InsertUpdate" nodes by creating or updating entities.

Parameters :
Name Type Optional Description
item XdbAction No
  • An XdbActions object containing rows of entity data.
Returns : any

A promise that resolves when all InsertUpdate nodes are processed.

Private Async processMediaNodes
processMediaNodes(item: XdbAction)

Process "media" nodes by creating or updating a media object.

Parameters :
Name Type Optional Description
item XdbAction No
  • An XdbActions object containing rows of media data.
Returns : any

A promise that resolves when all media nodes are processed.

Private Async processQueryNodes
processQueryNodes(item: XdbAction)

Process "query" nodes by execute SQL queries.

Parameters :
Name Type Optional Description
item XdbAction No
  • An XdbActions object containing rows of file data.
Returns : any

A promise that resolves when all query nodes are processed.

Private Async processReadOperators
processReadOperators(rows: Array<XdbRowData | FileRow>)

Processes @read operators in attributes and nodes, downloads the file along the path contained in the ${@read:/path} construct, replaces this construct with its contents

Parameters :
Name Type Optional Description
rows Array<XdbRowData | FileRow> No
  • A list of xml-row object containing the name property.
Returns : any
Private Async processRemoveNodes
processRemoveNodes(item: XdbAction)

Process "remove" nodes by removing the specified entities.

Parameters :
Name Type Optional Description
item XdbAction No
  • An XdbActions object containing rows of entity data to remove.
Returns : any

A promise that resolves when all remove nodes are processed.

Private Async setEntityPropertiesFromRowData
setEntityPropertiesFromRowData(entity: any, repository: Repository, rowData: literal type)

Set entity properties from the provided row data.

Parameters :
Name Type Optional Description
entity any No
  • The entity to set properties for.
repository Repository<any> No
  • A TypeORM Repository instance for the entity.
rowData literal type No
  • An object containing row data.
Returns : unknown

The entity with properties set from rowData.

Private Async updateEntityFromRowData
updateEntityFromRowData(existingEntity: any, repository: Repository, rowData: literal type)

Update an existing entity with properties from the provided row data.

Parameters :
Name Type Optional Description
existingEntity any No
  • The existing entity to update.
repository Repository<any> No
  • A TypeORM Repository instance for the entity.
rowData literal type No
  • An object containing row data.
Returns : unknown

The updated entity.

Accessors

connection
getconnection()
import { Inject, Injectable, Logger } from "@nestjs/common";
import { InjectDataSource } from "@nestjs/typeorm";
import { DataSource, EntityMetadata, In, Repository } from "typeorm";
import { ColumnMetadata } from "typeorm/metadata/ColumnMetadata";

import * as AdmZip from "adm-zip";
import * as fs from "fs";
import * as path from "path";
import { KpConfig } from "../../../gen-src/kp.config";
import { FilesUtils } from "../../shared/utils/files.utils";
import { Xdb, XdbImportService } from "./xml-data-bridge.constants";
import { NumberUtils } from "../../shared/utils/number.utils";
import { LOGGER } from "../../shared/modules/log/log.constants";
import { Media, MediaManager } from "../media/media.types";
import { FileManager } from "../file/file.constants";
import { CacheService } from "../../shared/modules/cache/cache.types";
import { ExplorerService } from "../explorer/explorer.types";
import {
  FileRow,
  MediaRow,
  XdbAction,
  XdbObject,
  XdbRowData,
} from "./xml-data-bridge.types";
import { File } from "../file/file.types";
import { LocalizedStringEntity } from "../../shared/modules/locale/entity/localized-string.entity";
import readFile = FilesUtils.readFile;
import createDirectoriesIfNotExist = FilesUtils.createDirectoriesIfNotExist;
import readDirectoryRecursively = FilesUtils.readDirectoryRecursively;
import ReadOperatorRe = Xdb.ReadOperatorRe;
import generateRandomInt = NumberUtils.generateRandomInt;

/**
 * XmlDataBridgeImportService is responsible for importing data through XML.
 */
@Injectable()
export class XmlDataBridgeImportService extends XdbImportService {
  constructor(
    @InjectDataSource()
    private readonly dataSource: DataSource,
    @Inject(LOGGER) private readonly logger: Logger,
    private readonly mediaService: MediaManager,
    private readonly filesService: FileManager,
    private readonly cacheService: CacheService,
    private readonly explorerService: ExplorerService,
  ) {
    super();
  }

  private get connection() {
    return this.dataSource.manager.connection;
  }

  /**
   * Import XML data from an XdbObject.
   * @param xml - The XdbObject containing the XML data.
   * @returns A promise that resolves to a boolean indicating whether the import was successful.
   */
  async importXml(xml: XdbObject): Promise<boolean> {
    for (const item of xml.schema) {
      await this.processReadOperators(item.rows);
      switch (item.action) {
        case "InsertUpdate":
          await this.processInsertUpdateNodes(item);
          break;
        case "Media":
          await this.processMediaNodes(item);
          break;
        case "File":
          await this.processFileNodes(item);
          break;
        case "Remove":
          await this.processRemoveNodes(item);
          break;
        case "Include":
          await this.processIncludeNodes(item);
          break;
        case "Query":
          await this.processQueryNodes(item);
          break;
      }
    }
    return true;
  }

  /**
   * Import XML data from Zip-archive.
   * @param fileData - The XdbObject containing the XML data.
   * @returns A promise that resolves to a boolean indicating whether the import was successful.
   */
  async importFromFile(fileData: Buffer) {
    // write archive
    const tmpDir =
      process.cwd() + (await this.cacheService.get(KpConfig.TMP_DIR));
    await createDirectoriesIfNotExist(tmpDir);
    const fileName = generateRandomInt();
    const filePath = `${tmpDir}/${fileName}.zip`;
    const operationDir = `${tmpDir}/${fileName}`;
    await fs.promises.writeFile(filePath, fileData);
    // work with archive
    const arch = new AdmZip(filePath);
    await createDirectoriesIfNotExist(operationDir);
    arch.extractAllTo(operationDir, true);
    const fileList = await readDirectoryRecursively(operationDir);
    const xmlFiles: string[] = [];
    for (const dir of Object.keys(fileList)) {
      if (!fileList[dir]?.length) {
        continue;
      }
      for (const file of fileList[dir].sort()) {
        const extractedFilePath = path.normalize(
          `${operationDir}/${dir}/${file}`,
        );
        if (!extractedFilePath.endsWith(".xml")) {
          continue;
        }
        const contentBuf = await readFile(extractedFilePath);
        const zipDir = operationDir.replace(process.cwd(), "");
        const contentStr = contentBuf.toString().replace(/@zip:/g, zipDir);
        const modifiedBuf = Buffer.from(contentStr);
        const xml = await Xdb.parseXmlFile(modifiedBuf);
        await this.importXml(xml);
        xmlFiles.push(extractedFilePath);
      }
    }
    return xmlFiles.length > 0;
  }

  /**
   * Process "file" nodes by creating or updating a file.
   * @param item - An XdbActions object containing rows of file data.
   * @returns A promise that resolves when all file nodes are processed.
   */
  private async processFileNodes(item: XdbAction) {
    const rows = item.rows as FileRow[];
    for (const row of rows) {
      let existedEntity: File;
      if (row.code) {
        existedEntity = await this.filesService.findByCode(row.code);
      }
      const filePath = process.cwd() + row.file;
      const buf = await readFile(path.normalize(filePath));
      const isPublic = String(row.public) === "true";
      const file = await this.filesService.createOrUpdateFile(
        buf,
        row.file.split(".").pop(),
        isPublic,
        row.code,
        existedEntity?.id,
        row.name,
      );
      this.logger.log(
        `${existedEntity ? `Update` : `Create`} file with ID ${file.id}`,
      );
    }
  }

  /**
   * Process "query" nodes by execute SQL queries.
   * @param item - An XdbActions object containing rows of file data.
   * @returns A promise that resolves when all query nodes are processed.
   */
  private async processQueryNodes(item: XdbAction) {
    this.logger.log(item.attrs.content);
    await this.connection.query(item.attrs.content);
  }

  /**
   * Process "file" nodes by creating or updating a file.
   * @param item - An XdbActions object containing rows of file data.
   * @returns A promise that resolves when all file nodes are processed.
   */
  private async processIncludeNodes(item: XdbAction) {
    this.logger.log(item.attrs.read);
    const filePath = process.cwd() + item.attrs.read;
    const buf = await readFile(path.normalize(filePath));
    const xml = await Xdb.parseXmlFile(buf);
    await this.importXml(xml);
  }

  /**
   * Process "media" nodes by creating or updating a media object.
   * @param item - An XdbActions object containing rows of media data.
   * @returns A promise that resolves when all media nodes are processed.
   */
  private async processMediaNodes(item: XdbAction) {
    const rows = item.rows as MediaRow[];
    for (const row of rows) {
      let existedEntity: Media;
      if (row.code) {
        existedEntity = await this.mediaService.findByCode(row.code);
      }
      const localizedStrings = await this.getLocalizedStrings(row);
      const filePath = process.cwd() + row.file;
      const buf = await readFile(path.normalize(filePath));
      const media = await this.mediaService.createOrUpdateMedia(
        buf,
        row.type,
        row.code,
        existedEntity?.id,
        localizedStrings,
      );
      this.logger.log(
        `${existedEntity ? `Update` : `Create`} media with ID ${media.id}`,
      );
    }
  }

  /**
   * Process "remove" nodes by removing the specified entities.
   * @param item - An XdbActions object containing rows of entity data to remove.
   * @returns A promise that resolves when all remove nodes are processed.
   */
  private async processRemoveNodes(item: XdbAction) {
    const repository = this.connection.getRepository(item.attrs.target);
    for (const rowData of item.rows) {
      const whereConditions = this.getRowDataWhereConditions(rowData);
      if (Object.keys(whereConditions).length > 0) {
        const entityToRemove = await repository.findOne({
          where: whereConditions,
        });
        if (entityToRemove) {
          await repository.remove(entityToRemove);
          this.logRemovedEntity(repository, whereConditions);
        } else {
          this.logger.warn(
            `Entity [${item.attrs.target}] with ${JSON.stringify(whereConditions)} not found, no removal performed`,
          );
        }
      } else {
        this.logger.warn(
          `Invalid row data for [${item.attrs.target}], no removal performed`,
        );
      }
    }
  }

  /**
   * Get the WHERE conditions for a row data object.
   * @param rowData - An XdbRowData object containing data for a row.
   * @returns An object containing WHERE conditions.
   */
  private getRowDataWhereConditions(rowData: XdbRowData | FileRow): object {
    const whereConditions = {};
    for (const key in rowData) {
      if (rowData[key].value) {
        whereConditions[key] = rowData[key].value;
      } else {
        whereConditions[key] = rowData[key];
      }
    }
    return whereConditions;
  }

  /**
   * Log that an entity has been removed.
   * @param repository - A TypeORM Repository instance for the entity.
   * @param whereConditions - An object containing WHERE conditions.
   */
  private logRemovedEntity(
    repository: Repository<any>,
    whereConditions: object,
  ) {
    const metadata = repository.metadata;
    const keyValuePairs = Object.entries(whereConditions)
      .map(([key, value]) => `${key}=${value}`)
      .join("; ");
    this.logger.log(`Removed [${metadata.targetName}] with ${keyValuePairs}`);
  }

  /**
   * Process "InsertUpdate" nodes by creating or updating entities.
   * @param item - An XdbActions object containing rows of entity data.
   * @returns A promise that resolves when all InsertUpdate nodes are processed.
   */
  private async processInsertUpdateNodes(item: XdbAction) {
    const repository = this.connection.getRepository(item.attrs.target);
    for (const rowData of item.rows) {
      const uniqueKeyFields = this.getUniqueKeyFields(repository, rowData);
      let existingEntity = null;
      if (Object.keys(uniqueKeyFields).length) {
        existingEntity = await repository.findOne({ where: uniqueKeyFields });
      }
      let entity;
      if (existingEntity) {
        if (this.hasPushMode(item)) {
          const target = await this.explorerService.getTargetData(
            existingEntity.constructor.name,
          );
          entity = await this.explorerService.getEntityData(
            target.entity.target,
            existingEntity[target.primaryColumn.property],
            undefined,
            { fullRelations: true },
          );
        } else {
          entity = existingEntity;
        }
        entity = await this.updateEntityFromRowData(
          entity,
          repository,
          rowData,
        );
      } else {
        entity = await this.createEntityFromRowData(repository, rowData);
      }
      await repository.save(entity);
      this.logSavedEntity(repository, entity, uniqueKeyFields, existingEntity);
    }
  }

  /**
   Checks whether an entity has a mode property with push value
   @param item - xml action-node
   */
  private hasPushMode(item: XdbAction) {
    for (const row of item.rows) {
      for (const key in row) {
        if (
          typeof row[key] === "object" &&
          row[key]?.hasOwnProperty("attrs") &&
          row[key]["attrs"]["mode"] === "push"
        ) {
          return true;
        }
      }
    }
    return false;
  }

  /**
   Log that an entity has been saved (created or updated).
   @param repository - A TypeORM Repository instance for the entity.
   @param entity - The saved entity.
   @param uniqueKeyFields - An object containing unique key fields.
   @param existingEntity - An existing entity, if any.
   */
  private logSavedEntity(
    repository: Repository<any>,
    entity: any,
    uniqueKeyFields: object,
    existingEntity: any,
  ) {
    const metadata = repository.metadata;
    const primaryKey = metadata.primaryColumns[0].propertyName;
    const keys = [primaryKey, ...Object.keys(uniqueKeyFields)];
    const values = [entity[primaryKey], ...Object.values(uniqueKeyFields)];
    const keyValuePairs = keys
      .map((key, index) => `${key}=${values[index]}`)
      .join("; ");
    this.logger.log(
      `${existingEntity ? `Update` : `Create`} [${metadata.targetName}] with ${keyValuePairs}`,
    );
  }

  /**
   Get the unique key fields for an entity based on the provided row data.
   @param repository - A TypeORM Repository instance for the entity.
   @param rowData - An object containing row data.
   @returns An object containing unique key fields.
   */
  private getUniqueKeyFields(
    repository: Repository<any>,
    rowData: { [key: string]: any },
  ): object {
    const entityMetadata = repository.metadata;
    const uniqueColumns = entityMetadata.columns.filter((column) => {
      if (this.isColumnUnique(entityMetadata, column)) {
        return true;
      }
      const uniqIndices = repository.metadata.indices.filter(
        (idc) => idc.isUnique,
      );
      for (const idc of uniqIndices) {
        const result = idc?.columns?.find(
          (col) => col.propertyName === column.propertyName,
        );
        if (result) {
          return true;
        }
      }
      return false;
    });
    const uniqueKeyFields = {};
    for (const uniqueColumn of uniqueColumns) {
      const propertyName = uniqueColumn.propertyName;
      if (rowData[propertyName]) {
        uniqueKeyFields[propertyName] =
          rowData[propertyName].value || rowData[propertyName];
      }
    }
    return uniqueKeyFields;
  }

  /**
   Set entity properties from the provided row data.
   @param entity - The entity to set properties for.
   @param repository - A TypeORM Repository instance for the entity.
   @param rowData - An object containing row data.
   @returns The entity with properties set from rowData.
   */
  private async setEntityPropertiesFromRowData(
    entity: any,
    repository: Repository<any>,
    rowData: { [key: string]: any },
  ) {
    for (const key in rowData) {
      const relation = repository.metadata.findRelationWithPropertyPath(key);
      if (relation && rowData[key].attrs) {
        if (rowData[key].value) {
          const relatedRepository = this.connection.getRepository(
            relation.type,
          );
          entity[key] = await relatedRepository.findOne({
            where: {
              [rowData[key].attrs.key]: rowData[key].value,
            },
          });
        } else if (rowData[key].values && rowData[key].attrs) {
          const relatedRepository = this.connection.getRepository(
            relation.inverseEntityMetadata.targetName,
          );
          const existedVal = entity[key];
          entity[key] = await relatedRepository.find({
            where: {
              [rowData[key].attrs.key]: In(rowData[key].values),
            },
          });
          if (rowData[key].attrs?.mode === "push" && existedVal?.length) {
            entity[key] = entity[key].concat(existedVal);
          }
        }
      } else if (rowData[key].value) {
        entity[key] = rowData[key].value;
      } else if (rowData[key].values) {
        entity[key] = rowData[key].values;
      } else {
        entity[key] = rowData[key];
        switch (entity[key]) {
          case "true":
            entity[key] = true;
            break;
          case "false":
            entity[key] = false;
            break;
          case "null":
            entity[key] = null;
            break;
        }
      }
    }
    return entity;
  }

  /**
   Update an existing entity with properties from the provided row data.
   @param existingEntity - The existing entity to update.
   @param repository - A TypeORM Repository instance for the entity.
   @param rowData - An object containing row data.
   @returns The updated entity.
   */
  private async updateEntityFromRowData(
    existingEntity: any,
    repository: Repository<any>,
    rowData: { [key: string]: any },
  ) {
    return this.setEntityPropertiesFromRowData(
      existingEntity,
      repository,
      rowData,
    );
  }

  /**
   Create a new entity with properties from the provided row data.
   @param repository - A TypeORM Repository instance for the entity.
   @param rowData - An object containing row data.
   @returns The created entity.
   */
  private async createEntityFromRowData(
    repository: Repository<any>,
    rowData: { [key: string]: any },
  ) {
    const entity = repository.create();
    return this.setEntityPropertiesFromRowData(entity, repository, rowData);
  }

  /**
   Determine if a column is unique.
   @param metadata - EntityMetadata for the entity.
   @param column - ColumnMetadata for the column.
   @returns A boolean indicating whether the column is unique.
   */
  private isColumnUnique(metadata: EntityMetadata, column: ColumnMetadata) {
    for (const uniq of metadata.uniques) {
      if (
        uniq.columns.find((col) => col.propertyName === column.propertyName) !==
        undefined
      ) {
        return true;
      }
    }
    return false;
  }

  /**
   Build localized strings object from xml-row
   @param row - A xml-row object containing the name property.
   @returns A promise that resolves to an array of LocalizedStringEntity objects.
   */
  private async getLocalizedStrings(row: {
    name: any;
  }): Promise<LocalizedStringEntity[]> {
    const localizedStrings: LocalizedStringEntity[] = [];
    if (row.name) {
      const rep = this.connection.getRepository(LocalizedStringEntity);
      for (const value of row.name.values) {
        const v = await rep.findOne({ where: { [row.name.attrs.key]: value } });
        localizedStrings.push(v);
      }
    }
    return localizedStrings;
  }

  /**
   Processes @read operators in attributes and nodes, downloads the file along the path contained
   in the ${@read:/path} construct, replaces this construct with its contents
   @param rows - A list of xml-row object containing the name property.
   */
  private async processReadOperators(rows: Array<XdbRowData | FileRow>) {
    for (const row of rows) {
      for (const val of Object.keys(row)) {
        let link: string = undefined;
        if (typeof row[val] === "string") {
          link = this.findReadOperator(row[val]);
          if (link) {
            const buf = await readFile(path.normalize(process.cwd() + link));
            row[val] = row[val].replace(ReadOperatorRe, buf.toString());
          }
        } else {
          link = this.findReadOperator(row[val].value);
          if (link) {
            const buf = await readFile(path.normalize(process.cwd() + link));
            row[val] = row[val].value.replace(ReadOperatorRe, buf.toString());
          }
        }
      }
    }
  }

  /**
   * Finds a read operator in a string.
   * @param input - The input string.
   * @returns The found read operator.
   */
  private findReadOperator(input: string) {
    const match = input?.match(ReadOperatorRe);
    if (match?.length > 1) {
      return match[1];
    }
    return undefined;
  }
}

results matching ""

    No results matching ""