You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

439 lines
8.6 KiB

MDEV-20377: Make WITH_MSAN more usable MemorySanitizer (clang -fsanitize=memory) requires that all code be compiled with instrumentation enabled. The C runtime library is an exception. Failure to use instrumented libraries will cause bogus messages about memory being uninitialized. In WITH_MSAN builds, we must avoid calling getservbyname(), because even though it is a standard library function, it is not instrumented, not even in clang 10. The following cmake options were tested: -DCMAKE_C_FLAGS='-march=native -O2' -DCMAKE_CXX_FLAGS='-stdlib=libc++ -march=native -O2' -DWITH_EMBEDDED_SERVER=OFF -DWITH_UNIT_TESTS=OFF -DCMAKE_BUILD_TYPE=Debug -DWITH_INNODB_{BZIP2,LZ4,LZMA,LZO,SNAPPY}=OFF -DPLUGIN_{ARCHIVE,TOKUDB,MROONGA,OQGRAPH,ROCKSDB,CONNECT,SPIDER}=NO -DWITH_SAFEMALLOC=OFF -DWITH_{ZLIB,SSL,PCRE}=bundled -DHAVE_LIBAIO_H=0 -DWITH_MSAN=ON MEM_MAKE_DEFINED(): An alias for VALGRIND_MAKE_MEM_DEFINED() and in the future, __msan_unpoison(). For now, neither MEM_MAKE_DEFINED() nor MEM_UNDEFINED() perform any action under MSAN. Enabling them will catch more bugs, but will also require some more fixes or work-arounds. Json_writer::add_double(): Work around a frequently occurring failure in optimizer tests, related to EXPLAIN FORMAT=JSON. dtoa(): Disable MSAN altogether. For some reason, this function is triggering a lot of trouble, especially when invoked for DBUG functions. The MDL default timeout is dd=86400 seconds, and for some reason it is claimed to be uninitialized. InnoDB: Define UNIV_DEBUG_VALGRIND also WITH_MSAN. ut_crc32_8_hw(), ut_crc32_64_low_hw(): Use the compiler built-in functions instead of inline assembler when building WITH_MSAN. This will require at least -msse4.2 when building for IA-32 or AMD64. The inline assembler would not be instrumented, and would thus cause bogus failures.
6 years ago
11 years ago
MDEV-20377: Make WITH_MSAN more usable MemorySanitizer (clang -fsanitize=memory) requires that all code be compiled with instrumentation enabled. The C runtime library is an exception. Failure to use instrumented libraries will cause bogus messages about memory being uninitialized. In WITH_MSAN builds, we must avoid calling getservbyname(), because even though it is a standard library function, it is not instrumented, not even in clang 10. The following cmake options were tested: -DCMAKE_C_FLAGS='-march=native -O2' -DCMAKE_CXX_FLAGS='-stdlib=libc++ -march=native -O2' -DWITH_EMBEDDED_SERVER=OFF -DWITH_UNIT_TESTS=OFF -DCMAKE_BUILD_TYPE=Debug -DWITH_INNODB_{BZIP2,LZ4,LZMA,LZO,SNAPPY}=OFF -DPLUGIN_{ARCHIVE,TOKUDB,MROONGA,OQGRAPH,ROCKSDB,CONNECT,SPIDER}=NO -DWITH_SAFEMALLOC=OFF -DWITH_{ZLIB,SSL,PCRE}=bundled -DHAVE_LIBAIO_H=0 -DWITH_MSAN=ON MEM_MAKE_DEFINED(): An alias for VALGRIND_MAKE_MEM_DEFINED() and in the future, __msan_unpoison(). For now, neither MEM_MAKE_DEFINED() nor MEM_UNDEFINED() perform any action under MSAN. Enabling them will catch more bugs, but will also require some more fixes or work-arounds. Json_writer::add_double(): Work around a frequently occurring failure in optimizer tests, related to EXPLAIN FORMAT=JSON. dtoa(): Disable MSAN altogether. For some reason, this function is triggering a lot of trouble, especially when invoked for DBUG functions. The MDL default timeout is dd=86400 seconds, and for some reason it is claimed to be uninitialized. InnoDB: Define UNIV_DEBUG_VALGRIND also WITH_MSAN. ut_crc32_8_hw(), ut_crc32_64_low_hw(): Use the compiler built-in functions instead of inline assembler when building WITH_MSAN. This will require at least -msse4.2 when building for IA-32 or AMD64. The inline assembler would not be instrumented, and would thus cause bogus failures.
6 years ago
  1. /* Copyright (C) 2014, 2020, MariaDB Corporation.
  2. This program is free software; you can redistribute it and/or modify
  3. it under the terms of the GNU General Public License as published by
  4. the Free Software Foundation; version 2 of the License.
  5. This program is distributed in the hope that it will be useful,
  6. but WITHOUT ANY WARRANTY; without even the implied warranty of
  7. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  8. GNU General Public License for more details.
  9. You should have received a copy of the GNU General Public License
  10. along with this program; if not, write to the Free Software
  11. Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */
  12. #include "mariadb.h"
  13. #include "sql_priv.h"
  14. #include "sql_string.h"
  15. #include "my_json_writer.h"
  16. void Json_writer::append_indent()
  17. {
  18. if (!document_start)
  19. output.append('\n');
  20. for (int i=0; i< indent_level; i++)
  21. output.append(' ');
  22. }
  23. void Json_writer::start_object()
  24. {
  25. fmt_helper.on_start_object();
  26. if (!element_started)
  27. start_element();
  28. output.append("{");
  29. indent_level+=INDENT_SIZE;
  30. first_child=true;
  31. element_started= false;
  32. document_start= false;
  33. }
  34. void Json_writer::start_array()
  35. {
  36. if (fmt_helper.on_start_array())
  37. return;
  38. if (!element_started)
  39. start_element();
  40. output.append("[");
  41. indent_level+=INDENT_SIZE;
  42. first_child=true;
  43. element_started= false;
  44. document_start= false;
  45. }
  46. void Json_writer::end_object()
  47. {
  48. indent_level-=INDENT_SIZE;
  49. if (!first_child)
  50. append_indent();
  51. first_child= false;
  52. output.append("}");
  53. }
  54. void Json_writer::end_array()
  55. {
  56. if (fmt_helper.on_end_array())
  57. return;
  58. indent_level-=INDENT_SIZE;
  59. if (!first_child)
  60. append_indent();
  61. output.append("]");
  62. }
  63. Json_writer& Json_writer::add_member(const char *name)
  64. {
  65. size_t len= strlen(name);
  66. if (fmt_helper.on_add_member(name, len))
  67. return *this; // handled
  68. // assert that we are in an object
  69. DBUG_ASSERT(!element_started);
  70. start_element();
  71. output.append('"');
  72. output.append(name, len);
  73. output.append("\": ", 3);
  74. return *this;
  75. }
  76. Json_writer& Json_writer::add_member(const char *name, size_t len)
  77. {
  78. if (fmt_helper.on_add_member(name, len))
  79. return *this; // handled
  80. // assert that we are in an object
  81. DBUG_ASSERT(!element_started);
  82. start_element();
  83. output.append('"');
  84. output.append(name, len);
  85. output.append("\": ");
  86. return *this;
  87. }
  88. /*
  89. Used by formatting helper to print something that is formatted by the helper.
  90. We should only separate it from the previous element.
  91. */
  92. void Json_writer::start_sub_element()
  93. {
  94. //element_started= true;
  95. if (first_child)
  96. first_child= false;
  97. else
  98. output.append(',');
  99. append_indent();
  100. }
  101. void Json_writer::start_element()
  102. {
  103. element_started= true;
  104. if (first_child)
  105. first_child= false;
  106. else
  107. output.append(',');
  108. append_indent();
  109. }
  110. void Json_writer::add_ll(longlong val)
  111. {
  112. char buf[64];
  113. my_snprintf(buf, sizeof(buf), "%lld", val);
  114. add_unquoted_str(buf);
  115. }
  116. void Json_writer::add_ull(ulonglong val)
  117. {
  118. char buf[64];
  119. my_snprintf(buf, sizeof(buf), "%llu", val);
  120. add_unquoted_str(buf);
  121. }
  122. /* Add a memory size, printing in Kb, Kb, Gb if necessary */
  123. void Json_writer::add_size(longlong val)
  124. {
  125. char buf[64];
  126. size_t len;
  127. if (val < 1024)
  128. len= my_snprintf(buf, sizeof(buf), "%lld", val);
  129. else if (val < 1024*1024*16)
  130. {
  131. /* Values less than 16MB are specified in KB for precision */
  132. len= my_snprintf(buf, sizeof(buf), "%lld", val/1024);
  133. strcpy(buf + len, "Kb");
  134. len+= 2;
  135. }
  136. else
  137. {
  138. len= my_snprintf(buf, sizeof(buf), "%lld", val/(1024*1024));
  139. strcpy(buf + len, "Mb");
  140. len+= 2;
  141. }
  142. add_str(buf, len);
  143. }
  144. void Json_writer::add_double(double val)
  145. {
  146. char buf[64];
  147. #if __has_feature(memory_sanitizer) // FIXME: remove this workaround for
  148. __msan_unpoison(&val, sizeof val); // main.range_mrr_icp & many other tests
  149. #endif
  150. size_t len= my_snprintf(buf, sizeof(buf), "%-.11lg", val);
  151. add_unquoted_str(buf, len);
  152. }
  153. void Json_writer::add_bool(bool val)
  154. {
  155. add_unquoted_str(val? "true" : "false");
  156. }
  157. void Json_writer::add_null()
  158. {
  159. add_unquoted_str("null", (size_t) 4);
  160. }
  161. void Json_writer::add_unquoted_str(const char* str)
  162. {
  163. size_t len= strlen(str);
  164. if (fmt_helper.on_add_str(str, len))
  165. return;
  166. if (!element_started)
  167. start_element();
  168. output.append(str, len);
  169. element_started= false;
  170. }
  171. void Json_writer::add_unquoted_str(const char* str, size_t len)
  172. {
  173. if (fmt_helper.on_add_str(str, len))
  174. return;
  175. if (!element_started)
  176. start_element();
  177. output.append(str, len);
  178. element_started= false;
  179. }
  180. void Json_writer::add_str(const char *str)
  181. {
  182. size_t len= strlen(str);
  183. if (fmt_helper.on_add_str(str, len))
  184. return;
  185. if (!element_started)
  186. start_element();
  187. output.append('"');
  188. output.append(str, len);
  189. output.append('"');
  190. element_started= false;
  191. }
  192. /*
  193. This function is used to add only num_bytes of str to the output string
  194. */
  195. void Json_writer::add_str(const char* str, size_t num_bytes)
  196. {
  197. if (fmt_helper.on_add_str(str, num_bytes))
  198. return;
  199. if (!element_started)
  200. start_element();
  201. output.append('"');
  202. output.append(str, num_bytes);
  203. output.append('"');
  204. element_started= false;
  205. }
  206. void Json_writer::add_str(const String &str)
  207. {
  208. add_str(str.ptr(), str.length());
  209. }
  210. Json_writer_temp_disable::Json_writer_temp_disable(THD *thd_arg)
  211. {
  212. thd= thd_arg;
  213. thd->opt_trace.disable_tracing_if_required();
  214. }
  215. Json_writer_temp_disable::~Json_writer_temp_disable()
  216. {
  217. thd->opt_trace.enable_tracing_if_required();
  218. }
  219. bool Single_line_formatting_helper::on_add_member(const char *name,
  220. size_t len)
  221. {
  222. DBUG_ASSERT(state== INACTIVE || state == DISABLED);
  223. if (state != DISABLED)
  224. {
  225. // remove everything from the array
  226. buf_ptr= buffer;
  227. //append member name to the array
  228. if (len < MAX_LINE_LEN)
  229. {
  230. memcpy(buf_ptr, name, len);
  231. buf_ptr+=len;
  232. *(buf_ptr++)= 0;
  233. line_len= owner->indent_level + (uint)len + 1;
  234. state= ADD_MEMBER;
  235. return true; // handled
  236. }
  237. }
  238. return false; // not handled
  239. }
  240. bool Single_line_formatting_helper::on_start_array()
  241. {
  242. if (state == ADD_MEMBER)
  243. {
  244. state= IN_ARRAY;
  245. return true; // handled
  246. }
  247. else
  248. {
  249. if (state != DISABLED)
  250. state= INACTIVE;
  251. // TODO: what if we have accumulated some stuff already? shouldn't we
  252. // flush it?
  253. return false; // not handled
  254. }
  255. }
  256. bool Single_line_formatting_helper::on_end_array()
  257. {
  258. if (state == IN_ARRAY)
  259. {
  260. flush_on_one_line();
  261. state= INACTIVE;
  262. return true; // handled
  263. }
  264. return false; // not handled
  265. }
  266. void Single_line_formatting_helper::on_start_object()
  267. {
  268. // Nested objects will not be printed on one line
  269. disable_and_flush();
  270. }
  271. bool Single_line_formatting_helper::on_add_str(const char *str,
  272. size_t len)
  273. {
  274. if (state == IN_ARRAY)
  275. {
  276. // New length will be:
  277. // "$string",
  278. // quote + quote + comma + space = 4
  279. if (line_len + len + 4 > MAX_LINE_LEN)
  280. {
  281. disable_and_flush();
  282. return false; // didn't handle the last element
  283. }
  284. //append string to array
  285. memcpy(buf_ptr, str, len);
  286. buf_ptr+=len;
  287. *(buf_ptr++)= 0;
  288. line_len += (uint)len + 4;
  289. return true; // handled
  290. }
  291. disable_and_flush();
  292. return false; // not handled
  293. }
  294. /*
  295. Append everything accumulated to the output on one line
  296. */
  297. void Single_line_formatting_helper::flush_on_one_line()
  298. {
  299. owner->start_sub_element();
  300. char *ptr= buffer;
  301. int nr= 0;
  302. while (ptr < buf_ptr)
  303. {
  304. char *str= ptr;
  305. if (nr == 0)
  306. {
  307. owner->output.append('"');
  308. owner->output.append(str);
  309. owner->output.append("\": ");
  310. owner->output.append('[');
  311. }
  312. else
  313. {
  314. if (nr != 1)
  315. owner->output.append(", ");
  316. owner->output.append('"');
  317. owner->output.append(str);
  318. owner->output.append('"');
  319. }
  320. nr++;
  321. while (*ptr!=0)
  322. ptr++;
  323. ptr++;
  324. }
  325. owner->output.append(']');
  326. /* We've printed out the contents of the buffer, mark it as empty */
  327. buf_ptr= buffer;
  328. }
  329. void Single_line_formatting_helper::disable_and_flush()
  330. {
  331. if (state == DISABLED)
  332. return;
  333. bool start_array= (state == IN_ARRAY);
  334. state= DISABLED;
  335. // deactivate ourselves and flush all accumulated calls.
  336. char *ptr= buffer;
  337. int nr= 0;
  338. while (ptr < buf_ptr)
  339. {
  340. char *str= ptr;
  341. size_t len= strlen(str);
  342. if (nr == 0)
  343. {
  344. owner->add_member(str, len);
  345. if (start_array)
  346. owner->start_array();
  347. }
  348. else
  349. {
  350. //if (nr == 1)
  351. // owner->start_array();
  352. owner->add_str(str, len);
  353. }
  354. nr++;
  355. ptr+= len+1;
  356. }
  357. buf_ptr= buffer;
  358. state= INACTIVE;
  359. }