17 17
# In order to get the warning messages to match up as expected, the candidate
18 18
# order here must much the target locale order in Python/pylifecycle.c
19 19
_C_UTF8_LOCALES = (
20 -
# Entries: (Target locale, target category, expected env var updates)
21 -
("C.UTF-8", "LC_ALL", "LC_ALL & LANG"),
22 -
("C.utf8", "LC_ALL", "LC_ALL & LANG"),
23 -
("UTF-8", "LC_CTYPE", "LC_CTYPE"),
20 +
# Entries: (Target locale, expected env var updates)
21 +
("C.UTF-8", "LC_CTYPE & LANG"),
22 +
("C.utf8", "LC_CTYPE & LANG"),
23 +
("UTF-8", "LC_CTYPE"),
24 24
)
25 25 26 26
# There's no reliable cross-platform way of checking locale alias
27 27
# lists, so the only way of knowing which of these locales will work
28 28
# is to try them with locale.setlocale(). We do that in a subprocess
29 29
# to avoid altering the locale of the test runner.
30 -
def _set_locale_in_subprocess(locale_name, category):
31 -
cmd_fmt = "import locale; print(locale.setlocale(locale.{}, '{}'))"
32 -
cmd = cmd_fmt.format(category, locale_name)
30 +
def _set_locale_in_subprocess(locale_name):
31 +
cmd_fmt = "import locale; print(locale.setlocale(locale.LC_CTYPE, '{}'))"
32 +
cmd = cmd_fmt.format(locale_name)
33 33
result, py_cmd = run_python_until_end("-c", cmd, __isolated=True)
34 34
return result.rc == 0
35 35 36 -
# Details of the CLI warning emitted at runtime
37 -
CLI_COERCION_WARNING_FMT = (
38 -
"Python detected LC_CTYPE=C: {} coerced to {} (set another locale "
39 -
"or PYTHONCOERCECLOCALE=0 to disable this locale coercion behavior)."
40 -
)
41 - 42 36
_EncodingDetails = namedtuple("EncodingDetails",
43 37
"fsencoding stdin_info stdout_info stderr_info")
44 38
@@ -67,9 +61,13 @@ def _handle_output_variations(data):
67 61
"""Adjust the output to handle platform specific idiosyncrasies
68 62 69 63
* Some platforms report ASCII as ANSI_X3.4-1968
64 +
* Some platforms report ASCII as US-ASCII
70 65
* Some platforms report UTF-8 instead of utf-8
71 66
"""
72 -
return data.replace(b"ANSI_X3.4-1968", b"ascii").lower()
67 +
data = data.replace(b"ANSI_X3.4-1968", b"ascii")
68 +
data = data.replace(b"US-ASCII", b"ascii")
69 +
data = data.lower()
70 +
return data
73 71 74 72
@classmethod
75 73
def get_child_details(cls, env_vars):
@@ -98,50 +96,116 @@ def get_child_details(cls, env_vars):
98 96
return child_encoding_details, stderr_lines
99 97 100 98 101 -
@test.support.cpython_only
102 -
@unittest.skipUnless(sysconfig.get_config_var("PY_COERCE_C_LOCALE"),
103 -
"C locale coercion disabled at build time")
104 -
class LocaleOverrideTests(unittest.TestCase):
99 +
class _ChildProcessEncodingTestCase(unittest.TestCase):
100 +
# Base class to check for expected encoding details in a child process
101 + 102 +
def _check_child_encoding_details(self,
103 +
env_vars,
104 +
expected_fsencoding,
105 +
expected_warning):
106 +
"""Check the C locale handling for the given process environment
107 + 108 +
Parameters:
109 +
expected_fsencoding: the encoding the child is expected to report
110 +
allow_c_locale: setting to use for PYTHONALLOWCLOCALE
111 +
None: don't set the variable at all
112 +
str: the value set in the child's environment
113 +
"""
114 +
result = EncodingDetails.get_child_details(env_vars)
115 +
encoding_details, stderr_lines = result
116 +
self.assertEqual(encoding_details,
117 +
EncodingDetails.get_expected_details(
118 +
expected_fsencoding))
119 +
self.assertEqual(stderr_lines, expected_warning)
120 + 121 +
# Details of the shared library warning emitted at runtime
122 +
LIBRARY_C_LOCALE_WARNING = (
123 +
"Python runtime initialized with LC_CTYPE=C (a locale with default ASCII "
124 +
"encoding), which may cause Unicode compatibility problems. Using C.UTF-8, "
125 +
"C.utf8, or UTF-8 (if available) as alternative Unicode-compatible "
126 +
"locales is recommended."
127 +
)
128 + 129 +
@unittest.skipUnless(sysconfig.get_config_var("PY_WARN_ON_C_LOCALE"),
130 +
"C locale runtime warning disabled at build time")
131 +
class LocaleWarningTests(_ChildProcessEncodingTestCase):
132 +
# Test warning emitted when running in the C locale
133 + 134 +
def test_library_c_locale_warning(self):
135 +
self.maxDiff = None
136 +
for locale_to_set in ("C", "POSIX", "invalid.ascii"):
137 +
var_dict = {
138 +
"LC_ALL": locale_to_set
139 +
}
140 +
with self.subTest(forced_locale=locale_to_set):
141 +
self._check_child_encoding_details(var_dict,
142 +
"ascii",
143 +
[LIBRARY_C_LOCALE_WARNING])
144 + 145 +
# Details of the CLI locale coercion warning emitted at runtime
146 +
CLI_COERCION_WARNING_FMT = (
147 +
"Python detected LC_CTYPE=C: {} coerced to {} (set another locale "
148 +
"or PYTHONCOERCECLOCALE=0 to disable this locale coercion behavior)."
149 +
)
150 + 151 +
class _LocaleCoercionTargetsTestCase(_ChildProcessEncodingTestCase):
152 +
# Base class for test cases that rely on coercion targets being defined
105 153 106 154
available_targets = []
155 +
targets_required = True
107 156 108 157
@classmethod
109 158
def setUpClass(cls):
110 159
first_target_locale = first_env_updates = None
111 160
available_targets = cls.available_targets
112 161
# Find the target locales available in the current system
113 -
for target_locale, target_category, env_updates in _C_UTF8_LOCALES:
114 -
if _set_locale_in_subprocess(target_locale, target_category):
162 +
for target_locale, env_updates in _C_UTF8_LOCALES:
163 +
if _set_locale_in_subprocess(target_locale):
115 164
available_targets.append(target_locale)
116 165
if first_target_locale is None:
117 166
first_target_locale = target_locale
118 167
first_env_updates = env_updates
119 -
if not available_targets:
168 +
if cls.targets_required and not available_targets:
120 169
raise unittest.SkipTest("No C-with-UTF-8 locale available")
121 170
# Expect coercion to use the first available locale
122 171
cls.EXPECTED_COERCION_WARNING = CLI_COERCION_WARNING_FMT.format(
123 172
first_env_updates, first_target_locale
124 173
)
125 174 126 -
def _check_child_encoding_details(self,
127 -
env_vars,
128 -
expected_fsencoding,
129 -
expected_warning):
130 -
"""Check the C locale handling for various configurations
131 175 132 -
Parameters:
133 -
expected_fsencoding: the encoding the child is expected to report
134 -
allow_c_locale: setting to use for PYTHONALLOWCLOCALE
135 -
None: don't set the variable at all
136 -
str: the value set in the child's environment
137 -
"""
138 -
result = EncodingDetails.get_child_details(env_vars)
139 -
encoding_details, stderr_lines = result
140 -
self.assertEqual(encoding_details,
141 -
EncodingDetails.get_expected_details(
142 -
expected_fsencoding))
143 -
self.assertEqual(stderr_lines, expected_warning)
176 +
class LocaleConfigurationTests(_LocaleCoercionTargetsTestCase):
177 +
# Test explicit external configuration via the process environment
144 178 179 +
def test_external_target_locale_configuration(self):
180 +
# Explicitly setting a target locale should give the same behaviour as
181 +
# is seen when implicitly coercing to that target locale
182 +
self.maxDiff = None
183 + 184 +
expected_warning = []
185 +
expected_fsencoding = "utf-8"
186 + 187 +
base_var_dict = {
188 +
"LANG": "",
189 +
"LC_CTYPE": "",
190 +
"LC_ALL": "",
191 +
}
192 +
for env_var in ("LANG", "LC_CTYPE"):
193 +
for locale_to_set in self.available_targets:
194 +
with self.subTest(env_var=env_var,
195 +
configured_locale=locale_to_set):
196 +
var_dict = base_var_dict.copy()
197 +
var_dict[env_var] = locale_to_set
198 +
self._check_child_encoding_details(var_dict,
199 +
expected_fsencoding,
200 +
expected_warning)
201 + 202 + 203 + 204 +
@test.support.cpython_only
205 +
@unittest.skipUnless(sysconfig.get_config_var("PY_COERCE_C_LOCALE"),
206 +
"C locale coercion disabled at build time")
207 +
class LocaleCoercionTests(_LocaleCoercionTargetsTestCase):
208 +
# Test implicit reconfiguration of the environment during CLI startup
145 209 146 210
def _check_c_locale_coercion(self, expected_fsencoding, coerce_c_locale):
147 211
"""Check the C locale handling for various configurations
@@ -165,7 +229,7 @@ def _check_c_locale_coercion(self, expected_fsencoding, coerce_c_locale):
165 229
"LC_CTYPE": "",
166 230
"LC_ALL": "",
167 231
}
168 -
for env_var in base_var_dict:
232 +
for env_var in ("LANG", "LC_CTYPE"):
169 233
for locale_to_set in ("", "C", "POSIX", "invalid.ascii"):
170 234
with self.subTest(env_var=env_var,
171 235
nominal_locale=locale_to_set,
@@ -178,7 +242,6 @@ def _check_c_locale_coercion(self, expected_fsencoding, coerce_c_locale):
178 242
expected_fsencoding,
179 243
expected_warning)
180 244 181 - 182 245
def test_test_PYTHONCOERCECLOCALE_not_set(self):
183 246
# This should coerce to the first available target locale by default
184 247
self._check_c_locale_coercion("utf-8", coerce_c_locale=None)
@@ -193,79 +256,13 @@ def test_PYTHONCOERCECLOCALE_set_to_zero(self):
193 256
# The setting "0" should result in the locale coercion being disabled
194 257
self._check_c_locale_coercion("ascii", coerce_c_locale="0")
195 258 196 -
def test_external_target_locale_configuration(self):
197 -
# Explicitly setting a target locale should give the same behaviour as
198 -
# is seen when implicitly coercing to that target locale
199 -
self.maxDiff = None
200 - 201 -
expected_warning = []
202 -
expected_fsencoding = "utf-8"
203 - 204 -
base_var_dict = {
205 -
"LANG": "",
206 -
"LC_CTYPE": "",
207 -
"LC_ALL": "",
208 -
}
209 -
for env_var in base_var_dict:
210 -
for locale_to_set in self.available_targets:
211 -
with self.subTest(env_var=env_var,
212 -
configured_locale=locale_to_set):
213 -
var_dict = base_var_dict.copy()
214 -
var_dict[env_var] = locale_to_set
215 -
self._check_child_encoding_details(var_dict,
216 -
expected_fsencoding,
217 -
expected_warning)
218 - 219 - 220 -
# Details of the shared library warning emitted at runtime
221 -
LIBRARY_C_LOCALE_WARNING = (
222 -
"Python runtime initialized with LC_CTYPE=C (a locale with default ASCII "
223 -
"encoding), which may cause Unicode compatibility problems. Using C.UTF-8, "
224 -
"C.utf8, or UTF-8 (if available) as alternative Unicode-compatible "
225 -
"locales is recommended.\n"
226 -
)
227 - 228 -
@unittest.skipUnless(sysconfig.get_config_var("PY_WARN_ON_C_LOCALE"),
229 -
"C locale runtime warning disabled at build time")
230 -
class EmbeddingTests(unittest.TestCase):
231 -
def setUp(self):
232 -
here = os.path.abspath(__file__)
233 -
basepath = os.path.dirname(os.path.dirname(os.path.dirname(here)))
234 -
self.test_exe = exe = os.path.join(basepath, "Programs", "_testembed")
235 -
if not os.path.exists(exe):
236 -
self.skipTest("%r doesn't exist" % exe)
237 -
# This is needed otherwise we get a fatal error:
238 -
# "Py_Initialize: Unable to get the locale encoding
239 -
# LookupError: no codec search functions registered: can't find encoding"
240 -
self.addCleanup(os.chdir, os.getcwd())
241 -
os.chdir(basepath)
242 - 243 -
def run_embedded_interpreter(self, *args):
244 -
"""Runs a test in the embedded interpreter"""
245 -
cmd = [self.test_exe]
246 -
cmd.extend(args)
247 -
p = subprocess.Popen(cmd,
248 -
stdout=subprocess.PIPE,
249 -
stderr=subprocess.PIPE,
250 -
universal_newlines=True)
251 -
(out, err) = p.communicate()
252 -
self.assertEqual(p.returncode, 0,
253 -
"bad returncode %d, stderr is %r" %
254 -
(p.returncode, err))
255 -
return out, err
256 - 257 -
def test_library_c_locale_warning(self):
258 -
# Checks forced configuration of embedded interpreter IO streams
259 -
out, err = self.run_embedded_interpreter("c_locale_warning")
260 -
if test.support.verbose > 1:
261 -
print()
262 -
print(out)
263 -
print(err)
264 -
self.assertEqual(out, "")
265 -
self.assertEqual(err, LIBRARY_C_LOCALE_WARNING)
266 259 267 260
def test_main():
268 -
test.support.run_unittest(LocaleOverrideTests, EmbeddingTests)
261 +
test.support.run_unittest(
262 +
LocaleConfigurationTests,
263 +
LocaleCoercionTests,
264 +
LocaleWarningTests
265 +
)
269 266
test.support.reap_children()
270 267 271 268
if __name__ == "__main__":
RetroSearch is an open source project built by @garambo | Open a GitHub Issue
Search and Browse the WWW like it's 1997 | Search results from DuckDuckGo
HTML:
3.2
| Encoding:
UTF-8
| Version:
0.7.4