Artificial Intelligence Recommendation:Operator fitur kustom

Konfigurasi

{
    "feature_name": "my_custom_fg_op",
    "feature_type": "custom_feature",
    "operator_name": "EditDistance",
    "operator_lib_file": "libedit_distance.so",
    "expression": [
        "user:query",
        "item:title"
    ],
    "value_type": "string",
    "separator": ",",
    "default_value": "-1",
    "value_dimension": 1,
    "normalizer": "method=expression,expr=x>16?16:x",
    "num_buckets": 10000,
    "stub_type": false,
    "is_sequence": false,
    "is_op_thread_safe": true,
    ...
}

Tambahkan item konfigurasi lain sesuai kebutuhan. Seluruh string JSON diteruskan ke operator kustom.

Catatan tambahan:

• Item konfigurasi yang ditentukan pengguna tidak boleh memiliki nama yang sama dengan item konfigurasi framework.

  • Operator kustom dapat membaca dan menggunakan item konfigurasi yang ditentukan oleh framework. Namun, mencoba mengubah semantiknya akan menyebabkan perilaku tidak terdefinisi.

• Item konfigurasi yang bergantung pada file resource eksternal harus diakhiri dengan _file.

  • Penanda ini digunakan untuk menyinkronkan file resource untuk task offline.

Contoh konfigurasi

{
    "feature_name": "time_diff_seq",
    "feature_type": "custom_feature",
    "operator_name": "SeqExpr",
    "expression": ["user:cur_time", "user:clk_time_seq"],
    "formula": "cur_time - clk_time_seq",
    "default_value": "0",
    "value_type": "int32",
    "is_sequence": true,
    "num_buckets": 1000,
    "is_op_thread_safe": false
},
{
    "feature_name": "spherical_distance",
    "feature_type": "custom_feature",
    "operator_name": "SeqExpr",
    "expression": ["item:click_id_lng", "item:click_id_lat", "user:j_lng", "user:j_lat"],
    "formula": "spherical_distance",
    "default_value": "0",
    "value_type": "double",
    "is_sequence": true,
    "is_op_thread_safe": true,
    "value_dimension": 1,
    "normalizer": "method=expression,expr=sqrt(x)"
}

• formula: Sebuah ekspresi. Untuk informasi selengkapnya tentang ekspresi yang didukung, lihat expr_feature.

  • spherical_distance: Menghitung jarak antara dua koordinat lintang dan bujur. Parameter adalah [lng1_seq, lat1_seq, lng2, lat2]. Dua parameter pertama adalah sequence, dan dua terakhir adalah nilai scalar.

  Ini adalah contoh Sequence feature kustom dalam format tiled. Untuk contoh Sequence feature kustom dalam format nested, lihat sequence_feature.

Antarmuka C++

#pragma once
#ifndef FEATURE_GENERATOR_PLUGIN_BASE_H
#define FEATURE_GENERATOR_PLUGIN_BASE_H

#include <absl/container/flat_hash_map.h>
#include <absl/strings/string_view.h>
#include <absl/types/optional.h>

#include <stdexcept>
#include <utility>
#include <vector>

#include "fsmap.h"
#include "integral_types.h"

namespace fg {

using absl::optional;
using std::string;
using std::vector;

template <typename T>
using List = std::vector<T>;
template <typename K, typename V>
using Map = absl::flat_hash_map<K, V>;
template <typename K, typename V>
using MapArray = std::vector<std::pair<K, V>>;
using Matrix = std::vector<std::vector<float>>;
using MatrixL = std::vector<std::vector<int64>>;
using MatrixS = std::vector<std::vector<string>>;
template <typename K, typename V>
using FSMap = featurestore::type::fs_map<K, V>;

using FieldPtr = absl::variant<
    const optional<string>*, const optional<int32>*, const optional<int64>*,
    const optional<float>*, const optional<double>*,
    const optional<absl::string_view>*,

    const List<string>*, const List<int32>*, const List<int64>*,
    const List<float>*, const List<double>*, const List<absl::string_view>*,

    const Map<string, string>*, const Map<string, int32>*,
    const Map<string, int64>*, const Map<string, float>*,
    const Map<string, double>*, const Map<string, absl::string_view>*,

    const Map<absl::string_view, absl::string_view>*,
    const Map<absl::string_view, int32>*, const Map<absl::string_view, int64>*,
    const Map<absl::string_view, float>*, const Map<absl::string_view, double>*,
    const Map<absl::string_view, string>*,

    const Map<int32, string>*, const Map<int32, int32>*,
    const Map<int32, int64>*, const Map<int32, float>*,
    const Map<int32, double>*, const Map<int32, absl::string_view>*,

    const Map<int64, string>*, const Map<int64, float>*,
    const Map<int64, double>*, const Map<int64, int32>*,
    const Map<int64, int64>*, const Map<int64, absl::string_view>*,

    const FSMap<absl::string_view, absl::string_view>*,
    const FSMap<absl::string_view, int32>*,
    const FSMap<absl::string_view, int64>*,
    const FSMap<absl::string_view, float>*,
    const FSMap<absl::string_view, double>*,

    const FSMap<int32, int32>*, const FSMap<int32, int64>*,
    const FSMap<int32, float>*, const FSMap<int32, double>*,
    const FSMap<int32, absl::string_view>*,

    const FSMap<int64, float>*, const FSMap<int64, double>*,
    const FSMap<int64, int32>*, const FSMap<int64, int64>*,
    const FSMap<int64, absl::string_view>*,

    const MapArray<string, string>*, const MapArray<string, int32>*,
    const MapArray<string, int64>*, const MapArray<string, float>*,
    const MapArray<string, double>*,

    const MapArray<int32, string>*, const MapArray<int32, float>*,
    const MapArray<int32, double>*, const MapArray<int32, int32>*,
    const MapArray<int32, int64>*,

    const MapArray<int64, string>*, const MapArray<int64, float>*,
    const MapArray<int64, double>*, const MapArray<int64, int32>*,
    const MapArray<int64, int64>*, const Matrix*, const MatrixL*,
    const MatrixS*>;

// represents a COLUMN of the feature table
using VariantVector = absl::variant<
    vector<optional<string>>, vector<optional<int32>>, vector<optional<int64>>,
    vector<optional<float>>, vector<optional<double>>,
    vector<optional<absl::string_view>>,

    vector<List<string>>, vector<List<int32>>, vector<List<int64>>,
    vector<List<float>>, vector<List<double>>, vector<List<absl::string_view>>,

    vector<Map<string, string>>, vector<Map<string, int32>>,
    vector<Map<string, int64>>, vector<Map<string, float>>,
    vector<Map<string, double>>, vector<Map<string, absl::string_view>>,

    vector<Map<absl::string_view, absl::string_view>>,
    vector<Map<absl::string_view, int32>>,
    vector<Map<absl::string_view, int64>>,
    vector<Map<absl::string_view, float>>,
    vector<Map<absl::string_view, double>>,

    vector<Map<int32, string>>, vector<Map<int32, int32>>,
    vector<Map<int32, int64>>, vector<Map<int32, float>>,
    vector<Map<int32, double>>, vector<Map<int32, absl::string_view>>,

    vector<Map<int64, string>>, vector<Map<int64, float>>,
    vector<Map<int64, double>>, vector<Map<int64, int32>>,
    vector<Map<int64, int64>>, vector<Map<int64, absl::string_view>>,

    vector<FSMap<absl::string_view, absl::string_view>>,
    vector<FSMap<absl::string_view, int32>>,
    vector<FSMap<absl::string_view, int64>>,
    vector<FSMap<absl::string_view, float>>,
    vector<FSMap<absl::string_view, double>>,

    vector<FSMap<int32, int32>>, vector<FSMap<int32, int64>>,
    vector<FSMap<int32, float>>, vector<FSMap<int32, double>>,
    vector<FSMap<int32, absl::string_view>>,

    vector<FSMap<int64, float>>, vector<FSMap<int64, double>>,
    vector<FSMap<int64, int32>>, vector<FSMap<int64, int64>>,
    vector<FSMap<int64, absl::string_view>>,

    vector<MapArray<string, string>>, vector<MapArray<string, int32>>,
    vector<MapArray<string, int64>>, vector<MapArray<string, float>>,
    vector<MapArray<string, double>>,

    vector<MapArray<int32, string>>, vector<MapArray<int32, float>>,
    vector<MapArray<int32, double>>, vector<MapArray<int32, int32>>,
    vector<MapArray<int32, int64>>,

    vector<MapArray<int64, string>>, vector<MapArray<int64, float>>,
    vector<MapArray<int64, double>>, vector<MapArray<int64, int32>>,
    vector<MapArray<int64, int64>>, vector<Matrix>, vector<MatrixL>,
    vector<MatrixS>>;

/**
 * @brief The public base class for custom feature operators.
 *
 * The framework checks if a subclass overrides the `BatchProcess` method. If it is overridden,
 * the framework calls this method to perform the feature transformation.
 * Otherwise, the framework selects one of the `ProcessWith*` methods to execute based on the `value_type` configuration.
 * You must implement the method that corresponds to the required output type.
 */
class IFeatureOP {
 public:
  class NotOverriddenException : public std::exception {
   public:
    explicit NotOverriddenException(std::string msg) : msg_(std::move(msg)) {}
    const char* what() const noexcept override {
      if (msg_.empty()) {
        return "unimplemented method called";
      }
      // Cache the message to a member variable to ensure that the returned pointer remains valid.
      cached_ = "unimplemented method called: " + msg_;
      return cached_.c_str();
    }

   private:
    std::string msg_;
    mutable std::string cached_;
  };

  virtual ~IFeatureOP() = default;

  /**
   * @brief The initialization method.
   * @param feature_config is a json string,
   * @return Returns 0 if the model is loaded successfully. Otherwise, it indicates that the model failed to load.
   */
  virtual int Initialize(const string& feature_config) = 0;

  /**
   * @brief Performs feature transformation and outputs the results as the string type.
   * @param inputs A record that can contain multiple fields.
   * @param outputs The outputs of the feature transformation.
   * @return A status code. A value of 0 indicates successful execution.
   */
  virtual int ProcessWithStrOutputs(const vector<FieldPtr>& inputs,
                                    vector<string>& outputs) {
    throw NotOverriddenException("ProcessWithStrOutputs(FieldPtr)");
  }

  /**
   * @brief Performs feature transformation and outputs the results as the int32 type.
   * @param inputs A record that can contain multiple fields.
   * @param outputs The outputs of the feature transformation.
   * @return A status code. A value of 0 indicates successful execution.
   */
  virtual int ProcessWithInt32Outputs(const vector<FieldPtr>& inputs,
                                      vector<int32>& outputs) {
    throw NotOverriddenException("ProcessWithInt32Outputs(FieldPtr)");
  }

  /**
   * @brief Performs feature transformation and outputs the results as the int64 type.
   * @param inputs A record that can contain multiple fields.
   * @param outputs The outputs of the feature transformation.
   * @return A status code. A value of 0 indicates successful execution.
   */
  virtual int ProcessWithInt64Outputs(const vector<FieldPtr>& inputs,
                                      vector<int64>& outputs) {
    throw NotOverriddenException("ProcessWithInt64Outputs(FieldPtr)");
  }

  /**
   * @brief Performs feature transformation and outputs the results as the float type.
   * @param inputs A record that can contain multiple fields.
   * @param outputs The outputs of the feature transformation.
   * @return A status code. A value of 0 indicates successful execution.
   */
  virtual int ProcessWithFloatOutputs(const vector<FieldPtr>& inputs,
                                      vector<float>& outputs) {
    throw NotOverriddenException("ProcessWithFloatOutputs(FieldPtr)");
  }

  /**
   * @brief Performs feature transformation and outputs the results as the double type.
   * @param inputs A record that can contain multiple fields.
   * @param outputs The outputs of the feature transformation.
   * @return A status code. A value of 0 indicates successful execution.
   */
  virtual int ProcessWithDoubleOutputs(const vector<FieldPtr>& inputs,
                                       vector<double>& outputs) {
    throw NotOverriddenException("ProcessWithDoubleOutputs(FieldPtr)");
  }

  /**
   * @brief An optional batch interface for processing multiple records.
   *
   * @param inputs A vector of input columns. `VariantVector` represents a feature column.
   * @param outputs
   * The transformed features. This method supports complex output types that can be used as inputs for other feature transformations.
   * @return A status code. A value of 0 indicates successful execution.
   */
  virtual int BatchProcess(const vector<VariantVector>& inputs,
                           VariantVector& outputs) {
    throw NotOverriddenException("BatchProcess");
  }

  /**
   * @brief Explicitly declares whether the subclass implements the BatchProcess method.
   *
   * The framework preferentially calls this method to check if `BatchProcess` has been overridden.
   * If your subclass implements `BatchProcess`, you must override this method to return true.
   * By default, it returns false to indicate that `BatchProcess` is not implemented.
   *
   * Note: This method is used to avoid exception propagation issues across dynamic library boundaries.
   * When a custom operator (a .so file) and the main program use different C++ ABIs or compilation options,
   * attempting to detect the implementation by calling `BatchProcess` and catching an exception may fail.
   *
   * @return true if the subclass implements BatchProcess.
   * @return false if the subclass does not implement BatchProcess (default).
   */
  virtual bool HasBatchProcessImpl() const { return false; }
};

using CreateOperatorFunc = IFeatureOP* (*)();

inline FieldPtr GetFieldPtr(const VariantVector& input, size_t i) {
  return absl::visit(
      [&](const auto& vec) -> FieldPtr {
        if (i >= vec.size()) {
          throw std::out_of_range("GetFieldPtr: index " + std::to_string(i) +
                                  " out of range [0, " +
                                  std::to_string(vec.size()) + ")");
        }
        return &vec.at(i);
      },
      input);
}
}  // namespace fg

#if defined(__GNUC__)
#define PLUGIN_API_HIDDEN \
  __attribute__((visibility("hidden"))) __attribute__((used))
#define PLUGIN_API_EXPORT \
  __attribute__((visibility("default"))) __attribute__((used))
#else
#define PLUGIN_API_HIDDEN
#define PLUGIN_API_EXPORT
#endif

std::vector<std::string>& getLocalNames();
std::vector<std::pair<std::string, void*>>& getLocalRegs();

#define REGISTER_PLUGIN(OpName, OpClass)                            \
  extern "C" PLUGIN_API_EXPORT fg::IFeatureOP* create##OpClass() {  \
    return new fg::OpClass();                                       \
  }                                                                 \
  namespace {                                                       \
  struct _Reg_##OpClass {                                           \
    _Reg_##OpClass() {                                              \
      getLocalNames().push_back(OpName);                            \
      getLocalRegs().emplace_back(OpName, (void*)&create##OpClass); \
    }                                                               \
  };                                                                \
  static _Reg_##OpClass _dummy_##OpClass __attribute__((used));     \
  }

#endif  // FEATURE_GENERATOR_PLUGIN_BASE_H

Panduan developer

• Unduh file dependensi API fg-api.tar.gz. File ini berisi header file yang diperlukan.

• Wariskan kelas dasar IFeatureOP, implementasikan metode Initialize, dan implementasikan setidaknya satu metode ProcessWith*.

• Kelas implementasi Anda harus menyertakan konstruktor tanpa parameter.

• Framework meneruskan string konfigurasi JSON ke metode Initialize. Uraikan item konfigurasi yang diperlukan dari string ini.

• Framework memanggil metode ProcessWith* yang sesuai berdasarkan item konfigurasi value_type. Jika metode untuk tipe yang sesuai tidak diimplementasikan, pengecualian waktu proses dilemparkan.

  • Metode ProcessWith* memproses satu record. Metode ini dapat memiliki beberapa bidang input dan output multi-dimensi, seperti fitur multi-nilai.

  • VariantRecord mendefinisikan semua tipe bidang fitur yang dapat diproses oleh framework.

  • Kode Anda harus mendukung sebanyak mungkin tipe dengan mengimplementasikan operasi transformasi fitur yang sesuai untuk setiap tipe input yang mungkin. Jika tipe tertentu tidak diperlukan, lemparkan pengecualian secara langsung.

  • FSMAP adalah tipe yang perlu didukung saat menggunakan featurestore. Tipe ini dapat meningkatkan performa processor secara signifikan.

• Implementasikan hanya operasi transformasi fitur sebelum operasi diskretisasi. Jika operasi diskretisasi dikonfigurasi, framework akan melakukannya secara otomatis.

• Gunakan makro REGISTER_PLUGIN untuk mendaftarkan operator fitur baru. Jika tidak, framework tidak dapat menggunakannya.

  • REGISTER_PLUGIN("OperatorName", OperatorClass): Ganti kedua parameter makro sesuai kebutuhan. Gunakan nama yang sama untuk kedua parameter.

  • Nilai untuk operator_name dalam item konfigurasi harus 'OperatorName'.

  • Daftarkan operator dalam file implementasi, bukan file header.

• Framework memindai semua pustaka tautan dinamis di direktori tertentu dan mencoba memuat operator fitur yang diperlukan saat diperlukan.

  • Gunakan variabel lingkungan FEATURE_OPERATOR_DIR untuk menentukan direktori tempat file pustaka tautan dinamis berada.

  • Setiap pustaka tautan dinamis dapat berisi implementasi beberapa operator fitur.

• Antarmuka BatchProcess digunakan untuk pemrosesan batch dan memproses satu batch data sekaligus.

  • Antarmuka ini opsional. Jika diimplementasikan, framework FG tidak lagi memanggil antarmuka granularitas sampel seperti ProcessWith*.

  • Setelah mengimplementasikan antarmuka ini, timpa fungsi bool HasBatchProcessImpl() const dan kembalikan true untuk menginstruksikan program utama menggunakan antarmuka ini.

  • Mengimplementasikan antarmuka ini dapat meningkatkan performa. Misalnya, jika fitur sisi pengguna hanya berisi satu sampel per permintaan, gunakan mekanisme broadcast untuk menghindari penguraian berulang fitur sisi pengguna untuk fitur silang.

  • Saat stub_type=true dikonfigurasi dan tidak ada operasi binning yang ditetapkan, antarmuka ini dapat mengembalikan tipe valid apa pun, seperti Map.

  • Tipe VariantVector yang dikembalikan oleh fungsi BatchProcess bergantung pada nilai is_sequence, value_dimension, dan value_type. Untuk informasi selengkapnya, lihat deskripsi item konfigurasi value_dimension.

  • Untuk contoh antarmuka pemrosesan batch, unduh dan tinjau RegexReplace.

• Dependensi pihak ketiga

  • abseil-cpp (Gunakan versi yang sama dengan framework FG.)

  • Pustaka pihak ketiga yang menjadi dependensi operator kustom harus dikompilasi dengan menyematkan kode sumber atau menggunakan tautan statis. Jangan bergantung pada pustaka tautan dinamis apa pun, karena hal ini dapat menyebabkan operator gagal dimuat.

Fitur sequence

Jika item konfigurasi is_sequence diatur menjadi true, perhatikan hal berikut:

• Sequence fitur jarang

  • Jika operator menghasilkan sequence fitur jarang, seperti sequence ID item_id yang sebelumnya dikunjungi, dan setiap elemen sequence adalah nilai tunggal, keluarkan tipe apa pun.

  • Jika operator menghasilkan sequence fitur jarang dan setiap elemen sequence dapat memiliki beberapa nilai, keluarkan hanya tipe string. Atur value_type menjadi string dan gunakan pemisah chr(29) untuk memisahkan beberapa nilai.

• Sequence fitur padat

  • Ketika Operator menghasilkan urutan fitur jarang, seperti vektor penyematan dari item yang diakses secara historis, atur value_dimension ke dimensi setiap elemen dalam urutan tersebut.

  • Jika elemen sequence adalah scalar, atur value_dimension menjadi 1.

  • Jika elemen sequence adalah vektor, atur value_dimension menjadi panjang vektor.

  • Jumlah nilai fitur yang dikeluarkan oleh operator harus merupakan kelipatan bulat dari value_dimension.

Daftar operator kustom

Item konfigurasi

• EditDistance

  • encoding: Encoding teks input. Opsi: utf-8, latin. Nilai default adalah latin.

Contoh developer

Contoh berikut menunjukkan cara menghitung jarak edit antara dua teks input. File header adalah edit_distance.h.

#pragma once
#include "api/base_op.h"

namespace fg {
namespace functor {
  class EditDistanceFunctor;
}

using std::string;
using std::vector;


/**
 * @brief Edit distance: Inputs two strings and outputs their text edit distance.
 */
class EditDistance : public IFeatureOP {
 public:
  int Initialize(const string& feature_config) override;

  /// @return A status code. 0 indicates successful execution.
  int ProcessWithStrOutputs(const vector<FieldPtr>& inputs,
                            vector<string>& outputs) override;

  /// @return A status code. 0 indicates successful execution.
  int ProcessWithInt32Outputs(const vector<FieldPtr>& inputs,
                              vector<int32>& outputs) override;

  /// @return A status code. 0 indicates successful execution.
  int ProcessWithInt64Outputs(const vector<FieldPtr>& inputs,
                              vector<int64>& outputs) override;

  /// @return A status code. 0 indicates successful execution.
  int ProcessWithFloatOutputs(const vector<FieldPtr>& inputs,
                              vector<float>& outputs) override;

  /// @return A status code. 0 indicates successful execution.
  int ProcessWithDoubleOutputs(const vector<FieldPtr>& inputs,
                               vector<double>& outputs) override;
 private:
  string feature_name_;
  std::unique_ptr<functor::EditDistanceFunctor> functor_p_;
};

}  // end of namespace fg

File implementasi adalah edit_distance.cc.

#include "edit_distance.h"

#include <absl/strings/ascii.h>
#include <absl/strings/str_join.h>

#include <nlohmann/json.hpp>
#include <numeric>  // Includes std::iota
#include <stdexcept>

#include "api/log.h"

namespace fg {
using absl::optional;

namespace functor {
template <class T>
int edit_distance(const T& s1, const T& s2) {
  int l1 = s1.size();
  int l2 = s2.size();
  if (l1 * l2 == 0) {
    return l1 + l2;
  }
  vector<int> prev(l2 + 1);
  vector<int> curr(l2 + 1);
  std::iota(prev.begin(), prev.end(), 0);
  for (int i = 0; i <= l1; ++i) {
    curr[0] = i;
    for (int j = 1; j <= l2; ++j) {
      int d = prev[j - 1];
      if (s1[i - 1] == s2[j - 1]) {
        curr[j] = d;
      } else {
        int d2 = std::min(prev[j], curr[j - 1]);
        curr[j] = 1 + std::min(d, d2);
      }
    }
    prev.swap(curr);
  }
  return prev[l2];
}

enum class Encoding : unsigned int { Latin = 0, UTF8 = 1 };

class EditDistanceFunctor {
 public:
  EditDistanceFunctor(const string& encoding) {
    string enc = absl::AsciiStrToLower(encoding);
    if (enc == "utf-8" || enc == "utf8") {
      encoding_ = Encoding::UTF8;
    } else {
      encoding_ = Encoding::Latin;
    }
  }

  int operator()(absl::string_view s1, absl::string_view s2) {
    if (encoding_ == Encoding::Latin) {
      return edit_distance(s1, s2);
    }
    if (encoding_ == Encoding::UTF8) {
      return edit_distance(from_bytes(s1), from_bytes(s2));
    }
    LOG(ERROR) << "EditDistanceFunctor found unsupported text encoding";
    assert(false);
    return 0;
  }

  const Encoding TextEncoding() const { return encoding_; }

 private:
  Encoding encoding_;

  std::wstring from_bytes(absl::string_view str) {
    std::wstring result;
    int i = 0;
    int len = (int)str.length();
    while (i < len) {
      int char_size = 0;
      int unicode = 0;

      if ((str[i] & 0x80) == 0) {
        unicode = str[i];
        char_size = 1;
      } else if ((str[i] & 0xE0) == 0xC0) {
        unicode = str[i] & 0x1F;
        char_size = 2;
      } else if ((str[i] & 0xF0) == 0xE0) {
        unicode = str[i] & 0x0F;
        char_size = 3;
      } else if ((str[i] & 0xF8) == 0xF0) {
        unicode = str[i] & 0x07;
        char_size = 4;
      } else {
        // Invalid UTF-8 sequence
        ++i;
        continue;
      }

      for (int j = 1; j < char_size; ++j) {
        unicode = (unicode << 6) | (str[i + j] & 0x3F);
      }

      if (unicode <= 0xFFFF) {
        result += static_cast<wchar_t>(unicode);
      } else {
        // Handle surrogate pairs for characters outside the BMP
        unicode -= 0x10000;
        result += static_cast<wchar_t>((unicode >> 10) + 0xD800);
        result += static_cast<wchar_t>((unicode & 0x3FF) + 0xDC00);
      }
      i += char_size;
    }
    return result;
  }
};
}  // namespace functor

// Defines the overloaded class.
template <class... Ts>
struct overloaded : Ts... {
  using Ts::operator()...;
};
// Class template argument deduction guide (C++17).
template <class... Ts>
overloaded(Ts...) -> overloaded<Ts...>;

int EditDistance::Initialize(const string& feature_config) {
  nlohmann::json cfg;
  try {
    cfg = nlohmann::json::parse(feature_config);
  } catch (nlohmann::json::parse_error& ex) {
    LOG(ERROR) << "parse error at byte " << ex.byte;
    LOG(ERROR) << "config: " << feature_config;
    throw std::runtime_error("parse EditDistance config failed");
  }

  feature_name_ = cfg.at("feature_name");
  string encoding = cfg.value("encoding", "latin");
  functor_p_ = std::make_unique<functor::EditDistanceFunctor>(encoding);
  functor::Encoding enc = functor_p_->TextEncoding();
  encoding = (enc == functor::Encoding::UTF8) ? "UTF-8" : "Latin";
  LOG(INFO) << "feature <" << feature_name_ << "> with text encoding: " << encoding;
  return 0;
}

int EditDistance::ProcessWithInt32Outputs(const vector<FieldPtr>& inputs,
                                          vector<int32>& outputs) {
  outputs.clear();
  if (inputs.size() < 2) {
    outputs.push_back(0);
    return -1;  // invalid inputs
  }

  int d = absl::visit(
      overloaded{
          [this](const optional<string>* s1, const optional<string>* s2) {
            absl::string_view empty_view;
            return functor_p_->operator()(*s1 ? **s1 : empty_view, *s2 ? **s2 : empty_view);
          },
          [this](const optional<absl::string_view>* s1,
                 const optional<absl::string_view>* s2) {
            absl::string_view empty_view;
            return functor_p_->operator()(*s1 ? **s1 : empty_view, *s2 ? **s2 : empty_view);
          },
          [this](const optional<absl::string_view>* s1,
                 const optional<string>* s2) {
            absl::string_view empty_view;
            return functor_p_->operator()(*s1 ? **s1 : empty_view, *s2 ? **s2 : empty_view);
          },
          [this](const optional<string>* s1,
                 const optional<absl::string_view>* s2) {
            absl::string_view empty_view;
            return functor_p_->operator()(*s1 ? **s1 : empty_view, *s2 ? **s2 : empty_view);
          },
          [this](const List<string>* s1, const List<string>* s2) {
            string str1 = absl::StrJoin(*s1, "");
            string str2 = absl::StrJoin(*s2, "");
            return functor_p_->operator()(str1, str2);
          },
          [this](const List<absl::string_view>* s1,
                 const List<absl::string_view>* s2) {
            string str1 = absl::StrJoin(*s1, "");
            string str2 = absl::StrJoin(*s2, "");
            return functor_p_->operator()(str1, str2);
          },
          [this](const auto* x, const auto* y) {
            ERROR_EXIT(feature_name_,
                       "unsupported input type: ", typeid(*x).name(), " vs ",
                       typeid(*y).name());
            return 0;
          }},
      inputs.at(0), inputs.at(1));
  outputs.push_back(d);
  return 0;
}

int EditDistance::ProcessWithInt64Outputs(const vector<FieldPtr>& inputs,
                                          vector<int64>& outputs) {
  vector<int32> distances;
  int status = ProcessWithInt32Outputs(inputs, distances);
  if (0 != status) {
    return status;
  }
  outputs.clear();
  outputs.insert(outputs.end(), distances.begin(), distances.end());
  return 0;
}

int EditDistance::ProcessWithFloatOutputs(const vector<FieldPtr>& inputs,
                                          vector<float>& outputs) {
  vector<int32> distances;
  int status = ProcessWithInt32Outputs(inputs, distances);
  if (0 != status) {
    return status;
  }
  outputs.clear();
  outputs.insert(outputs.end(), distances.begin(), distances.end());
  return 0;
}

int EditDistance::ProcessWithDoubleOutputs(const vector<FieldPtr>& inputs,
                                           vector<double>& outputs) {
  vector<int32> distances;
  int status = ProcessWithInt32Outputs(inputs, distances);
  if (0 != status) {
    return status;
  }
  outputs.clear();
  outputs.insert(outputs.end(), distances.begin(), distances.end());
  return 0;
}

int EditDistance::ProcessWithStrOutputs(const vector<FieldPtr>& inputs,
                                        vector<string>& outputs) {
  vector<int32> distances;
  int status = ProcessWithInt32Outputs(inputs, distances);
  if (0 != status) {
    return status;
  }
  outputs.clear();
  outputs.reserve(distances.size());
  std::transform(distances.begin(), distances.end(),
                 std::back_inserter(outputs),
                 [](int32& x) { return std::to_string(x); });
  return 0;
}

}  // end of namespace fg

REGISTER_PLUGIN("EditDistance", EditDistance);

Unduh kode sumber dari tabel di atas dan jalankan skrip build.sh untuk mengompilasi dan menghasilkan operator FG.

Kompilasi operator kustom

Gunakan lingkungan kompilasi yang sama dengan framework FG, termasuk standar bahasa (C++17) dan opsi kompilasi. Gunakan image compiler resmi. Detail image tersedia dalam skrip build.sh.

• Image lingkungan compiler (CentOS 7): mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/feature_generator:centos7-0.1.1

• Image lingkungan compiler (rockylinux:8, kompatibel dengan CentOS 8): mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/feature_generator:0.1.1

• Secara default, ABI C++11 tidak digunakan. Untuk menggunakan ABI baru, atur _GLIBCXX_USE_CXX11_ABI=1. Dalam kasus ini, gunakan hanya image kedua (tag: 0.1.1), yang berbasis rockylinux:8.

• Pastikan operator kustom tidak menggunakan tautan dinamis untuk pustaka pihak ketiga. Gunakan tautan statis atau salin kode sumber ke dalam proyek untuk dikompilasi.

Untuk informasi selengkapnya, lihat file CMakeLists.txt dalam contoh developer.