text-utils.cc
4.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
// sherpa-onnx/csrc/text-utils.cc
//
// Copyright 2009-2011 Saarland University; Microsoft Corporation
// Copyright 2023 Xiaomi Corporation
#include "sherpa-onnx/csrc/text-utils.h"
#include <assert.h>
#include <algorithm>
#include <limits>
#include <sstream>
#include <string>
#include <unordered_map>
#include <vector>
// This file is copied/modified from
// https://github.com/kaldi-asr/kaldi/blob/master/src/util/text-utils.cc
namespace sherpa_onnx {
// copied from kaldi/src/util/text-util.cc
template <class T>
class NumberIstream {
public:
explicit NumberIstream(std::istream &i) : in_(i) {}
NumberIstream &operator>>(T &x) {
if (!in_.good()) return *this;
in_ >> x;
if (!in_.fail() && RemainderIsOnlySpaces()) return *this;
return ParseOnFail(&x);
}
private:
std::istream &in_;
bool RemainderIsOnlySpaces() {
if (in_.tellg() != std::istream::pos_type(-1)) {
std::string rem;
in_ >> rem;
if (rem.find_first_not_of(' ') != std::string::npos) {
// there is not only spaces
return false;
}
}
in_.clear();
return true;
}
NumberIstream &ParseOnFail(T *x) {
std::string str;
in_.clear();
in_.seekg(0);
// If the stream is broken even before trying
// to read from it or if there are many tokens,
// it's pointless to try.
if (!(in_ >> str) || !RemainderIsOnlySpaces()) {
in_.setstate(std::ios_base::failbit);
return *this;
}
std::unordered_map<std::string, T> inf_nan_map;
// we'll keep just uppercase values.
inf_nan_map["INF"] = std::numeric_limits<T>::infinity();
inf_nan_map["+INF"] = std::numeric_limits<T>::infinity();
inf_nan_map["-INF"] = -std::numeric_limits<T>::infinity();
inf_nan_map["INFINITY"] = std::numeric_limits<T>::infinity();
inf_nan_map["+INFINITY"] = std::numeric_limits<T>::infinity();
inf_nan_map["-INFINITY"] = -std::numeric_limits<T>::infinity();
inf_nan_map["NAN"] = std::numeric_limits<T>::quiet_NaN();
inf_nan_map["+NAN"] = std::numeric_limits<T>::quiet_NaN();
inf_nan_map["-NAN"] = -std::numeric_limits<T>::quiet_NaN();
// MSVC
inf_nan_map["1.#INF"] = std::numeric_limits<T>::infinity();
inf_nan_map["-1.#INF"] = -std::numeric_limits<T>::infinity();
inf_nan_map["1.#QNAN"] = std::numeric_limits<T>::quiet_NaN();
inf_nan_map["-1.#QNAN"] = -std::numeric_limits<T>::quiet_NaN();
std::transform(str.begin(), str.end(), str.begin(), ::toupper);
if (inf_nan_map.find(str) != inf_nan_map.end()) {
*x = inf_nan_map[str];
} else {
in_.setstate(std::ios_base::failbit);
}
return *this;
}
};
/// ConvertStringToReal converts a string into either float or double
/// and returns false if there was any kind of problem (i.e. the string
/// was not a floating point number or contained extra non-whitespace junk).
/// Be careful- this function will successfully read inf's or nan's.
template <typename T>
bool ConvertStringToReal(const std::string &str, T *out) {
std::istringstream iss(str);
NumberIstream<T> i(iss);
i >> *out;
if (iss.fail()) {
// Number conversion failed.
return false;
}
return true;
}
template bool ConvertStringToReal<float>(const std::string &str, float *out);
template bool ConvertStringToReal<double>(const std::string &str, double *out);
void SplitStringToVector(const std::string &full, const char *delim,
bool omit_empty_strings,
std::vector<std::string> *out) {
size_t start = 0, found = 0, end = full.size();
out->clear();
while (found != std::string::npos) {
found = full.find_first_of(delim, start);
// start != end condition is for when the delimiter is at the end
if (!omit_empty_strings || (found != start && start != end))
out->push_back(full.substr(start, found - start));
start = found + 1;
}
}
template <class F>
bool SplitStringToFloats(const std::string &full, const char *delim,
bool omit_empty_strings, // typically false
std::vector<F> *out) {
assert(out != nullptr);
if (*(full.c_str()) == '\0') {
out->clear();
return true;
}
std::vector<std::string> split;
SplitStringToVector(full, delim, omit_empty_strings, &split);
out->resize(split.size());
for (size_t i = 0; i < split.size(); ++i) {
// assume atof never fails
F f = 0;
if (!ConvertStringToReal(split[i], &f)) return false;
(*out)[i] = f;
}
return true;
}
// Instantiate the template above for float and double.
template bool SplitStringToFloats(const std::string &full, const char *delim,
bool omit_empty_strings,
std::vector<float> *out);
template bool SplitStringToFloats(const std::string &full, const char *delim,
bool omit_empty_strings,
std::vector<double> *out);
} // namespace sherpa_onnx