8000 Fix matching regex word-boundary (\b) in strings replace by davidwendt · Pull Request #9997 · rapidsai/cudf · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Fix matching regex word-boundary (\b) in strings replace #9997

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Jan 20, 2022
Merged

Fix matching regex word-boundary (\b) in strings replace #9997

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
0226fdf
Fix matching regex word-boundary (BOW) in strings replace
davidwendt Jan 6, 2022
bc14f6e
Merge branch 'branch-22.02' into bug-replace-regex-bow
davidwendt Jan 7, 2022
b957f46
add const and rename some variables
davidwendt Jan 7, 2022
bc85257
Merge branch 'branch-22.02' into bug-replace-regex-bow
davidwendt Jan 10, 2022
9b5cbe7
Merge branch 'branch-22.02' into bug-replace-regex-bow
davidwendt Jan 10, 2022
e01ecbf
Merge branch 'branch-22.02' into bug-replace-regex-bow
davidwendt Jan 12, 2022
80281b3
Merge branch 'branch-22.02' into bug-replace-regex-bow
davidwendt Jan 18, 2022
00ffb8d
move if() and while() open brace
davidwendt Jan 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 41 additions & 29 deletions cpp/src/strings/replace/replace_re.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2019-2021, NVIDIA CORPORATION.
* Copyright (c) 2019-2022, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -62,37 +62,49 @@ struct replace_regex_fn {
if (!d_chars) d_offsets[idx] = 0;
return;
}
auto const d_str = d_strings.element<string_view>(idx);
auto const nchars = d_str.length(); // number of characters in input string
auto nbytes = d_str.size_bytes(); // number of bytes in input string
auto mxn = maxrepl < 0 ? nchars : maxrepl; // max possible replaces for this string
auto in_ptr = d_str.data(); // input pointer (i)
auto out_ptr = d_chars ? d_chars + d_offsets[idx] : nullptr; // output pointer (o)
size_type lpos = 0;
int32_t begin = 0;
int32_t end = static_cast<int32_t>(nchars);

auto const d_str = d_strings.element<string_view>(idx);
auto nbytes = d_str.size_bytes(); // number of bytes in input string
auto mxn = maxrepl < 0 ? d_str.length() + 1 : maxrepl; // max possible replaces for this string
auto in_ptr = d_str.data(); // input pointer (i)
auto out_ptr = d_chars ? d_chars + d_offsets[idx] // output pointer (o)
: nullptr;
size_type last_pos = 0;
int32_t begin = 0; // these are for calling prog.find
int32_t end = -1; // matches final word-boundary if at the end of the string

// copy input to output replacing strings as we go
while (mxn-- > 0) // maximum number of replaces
{
if (prog.is_empty() || prog.find<stack_size>(idx, d_str, begin, end) <= 0)
break; // no more matches
auto spos = d_str.byte_offset(begin); // get offset for these
auto epos = d_str.byte_offset(end); // character position values
nbytes += d_repl.size_bytes() - (epos - spos); // compute new size
if (out_ptr) // replace
{ // i:bbbbsssseeee
out_ptr = copy_and_increment(out_ptr, in_ptr + lpos, spos - lpos); // o:bbbb
out_ptr = copy_string(out_ptr, d_repl); // o:bbbbrrrrrr
// out_ptr ---^
lpos = epos; // i:bbbbsssseeee
} // in_ptr --^
begin = end;
end = static_cast<int32_t>(nchars);
while (mxn-- > 0) { // maximum number of replaces

if (prog.is_empty() || prog.find<stack_size>(idx, d_str, begin, end) <= 0) {
break; // no more matches
}

auto const start_pos = d_str.byte_offset(begin); // get offset for these
auto const end_pos = d_str.byte_offset(end); // character position values
nbytes += d_repl.size_bytes() - (end_pos - start_pos); // and compute new size

if (out_ptr) { // replace:
// i:bbbbsssseeee
out_ptr = copy_and_increment(out_ptr, // ^
in_ptr + last_pos, // o:bbbb
start_pos - last_pos); // ^
out_ptr = copy_string(out_ptr, d_repl); // o:bbbbrrrrrr
// out_ptr ---^
last_pos = end_pos; // i:bbbbsssseeee
} // in_ptr --^

begin = end + (begin == end);
end = -1;
}
if (out_ptr) // copy the remainder
memcpy(out_ptr, in_ptr + lpos, d_str.size_bytes() - lpos); // o:bbbbrrrrrreeee
else

if (out_ptr) {
memcpy(out_ptr, // copy the remainder
in_ptr + last_pos, // o:bbbbrrrrrreeee
d_str.size_bytes() - last_pos); // ^ ^
} else {
d_offsets[idx] = static_cast<int32_t>(nbytes);
}
}
};

Expand Down
12 changes: 11 additions & 1 deletion cpp/tests/strings/replace_regex_tests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2019-2021, NVIDIA CORPORATION.
* Copyright (c) 2019-2022, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -145,6 +145,16 @@ TEST_F(StringsReplaceRegexTest, MultiReplacement)
CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(*results, input);
}

TEST_F(StringsReplaceRegexTest, WordBoundary)
{
cudf::test::strings_column_wrapper input({"aba bcd\naba", "zéz", "A1B2-é3", "e é"});
auto results =
cudf::strings::replace_re(cudf::strings_column_view(input), "\\b", cudf::string_scalar("X"));
cudf::test::strings_column_wrapper expected(
{"XabaX XbcdX\nXabaX", "XzézX", "XA1B2X-Xé3X", "XeX XéX"});
CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(*results, expected);
}

TEST_F(StringsReplaceRegexTest, Multiline)
{
auto const multiline = cudf::strings::regex_flags::MULTILINE;
Expand Down
0