Fangjun Kuang
Committed by GitHub

Fix reading tokens.txt on Windows (#448)

@@ -50,6 +50,8 @@ static std::unordered_map<std::string, int32_t> ReadTokens(std::istream &is) { @@ -50,6 +50,8 @@ static std::unordered_map<std::string, int32_t> ReadTokens(std::istream &is) {
50 iss >> id; 50 iss >> id;
51 } 51 }
52 52
  53 + // eat the trailing \r\n on windows
  54 + iss >> std::ws;
53 if (!iss.eof()) { 55 if (!iss.eof()) {
54 SHERPA_ONNX_LOGE("Error: %s", line.c_str()); 56 SHERPA_ONNX_LOGE("Error: %s", line.c_str());
55 exit(-1); 57 exit(-1);