萬盛學電腦網

您现在的位置: 萬盛學電腦網 >> 程式設計 >  >> 資料庫 >> SQL Server 教學

全文檢索噪音詞過濾:JavaScript 實作移除中英文停用詞

背景

SQL Server 全文檢索會忽略「噪音詞」(Noise Words),如英文 the, and, is、中文 的, 是, 在。若查詢字串全為噪音詞會拋出錯誤,應用層預先過濾可提升查詢穩定度。

完整 JavaScript 實作

<script language="javascript">
<!--
// 中英文噪音詞表 (可依需求擴充)
var noise_word_list_ch = new Array(
  "?", "about", "$", "1","2","3","4","5","6","7","8","9","0","_",
  "a","b","c","d","e","f","g","h","i","j","k","l","m","n","o",
  "p","q","r","s","t","u","v","w","x","y","z",
  "after","all","also","an","and","another","any","are","as","at",
  "be","because","been","before","being","between","both","but","by",
  "came","can","come","could","did","do","each","for","from","get",
  "got","had","has","have","he","her","here","him","himself","his",
  "how","if","in","into","is","it","like","make","many","me","might",
  "more","most","much","must","my","never","now","of","on","only",
  "or","other","our","out","over","said","same","see","should",
  "since","some","still","such","take","than","that","the","their",
  "them","then","there","these","they","this","those","through",
  "to","too","under","up","very","was","way","we","well","were",
  "what","where","which","while","who","with","would","you","your",
  "的","一","不","在","人","有","是","為","以","於","上","他",
  "而","後","之","來","及","了","因","下","可","到","由","這",
  "與","也","此","但","並","個","其","已","無","小","我","們",
  "起","最","再","今","去","好","只","又","或","很","亦","某",
  "把","那","你","乃","它"
);

function trim_str_key(inputVal) {
  var inputStr = inputVal.toString();
  while (inputStr.charAt(inputStr.length-1)==" " || inputStr.charAt(0)==" ") {
    if (inputStr.charAt(inputStr.length-1)==" ") 
      inputStr = inputStr.substring(0, inputStr.length-1);
    if (inputStr.charAt(0)==" ") 
      inputStr = inputStr.substring(1, inputStr.length);
  }
  return inputStr;
}

function is_ch_noise_word(str_key) {
  var key_word = trim_str_key(str_key).toLowerCase();
  for (var i=0; i<noise_word_list_ch.length; i++) {
    if (noise_word_list_ch[i] == key_word) return true;
  }
  return false;
}

function remove_noise_word(str_source) {
  var tmp_str = "", ch = "", str_out = "";
  str_source = trim_str_key(str_source);
  if (str_source.length == 0) return str_out;

  for (var i=0; i<str_source.length; i++) {
    ch = str_source.charAt(i);
    if (ch == " ") {  // 空格作為詞界
      if (!is_ch_noise_word(tmp_str) && tmp_str != "") {
        str_out += tmp_str + " ";
      }
      tmp_str = "";
    } else {
      tmp_str += ch;
    }
  }
  // 處理最後一個詞
  if (!is_ch_noise_word(tmp_str)) str_out += tmp_str;
  return trim_str_key(str_out);
}

// 測試範例
// var abc = "av n";
// alert(remove_noise_word(abc));  // 輸出 "av n" (若 av, n 不在噪音表中)
// -->
</script>

使用方式

var userQuery = "關於 產品 的 價格";
var cleaned = remove_noise_word(userQuery);  // 結果: "產品 價格"
/* 將 cleaned 送入 CONTAINS/FREETEXT 查詢 */

擴充建議

  • 噪音詞表建議從 SQL Server 安裝目錄 \MSSQL\FTData\noiseENU.txt / noiseCHS.txt 同步更新。
  • 可改為伺服端預存程序或 CLR 函數,避免前後端邏輯不一致。
  • 支援標點符號、全形/半形正規化。
copyright © 萬盛學電腦網 all rights reserved