Fix for bug #801217 Assertion `t1->result_range' in Gcalc_operation_reducer::end_couple.

We cannot cut a line from a polygon. So if the polygon fits the condition, and
        the intersection of a line and the polygon doesn't, we just skip the line.
        That rule wasn't applied if the line start was inside the polygon, which leaded
        to the assertion.

per-file comments:
  mysql-test/r/gis-precise.result
Fix for bug #801217 Assertion `t1->result_range' in Gcalc_operation_reducer::end_couple.
        test result updated.
  mysql-test/t/gis-precise.test
Fix for bug #801217 Assertion `t1->result_range' in Gcalc_operation_reducer::end_couple.
        test case added.
  sql/gcalc_tools.cc
Fix for bug #801217 Assertion `t1->result_range' in Gcalc_operation_reducer::end_couple.
        Don't mark the line as a border if it's inside a polygon that fits the result condition.

mysql-test/r/gis-precise.result

@@ -279,3 +279,12 @@ AsText(ST_UNION(MultiPolygonFromText('
                      ((2 2, 2 8, 8 8, 8 2, 2 2), (4 4, 4 6, 6 6, 6 4, 4 4)))'),
 MultiPolygonFr
 POLYGON((0 0,0 5,0.555555555555556 5,1 9,2 8,8 8,8 2,7 2,5.33333333333333 2,3 1.125,3 0,0 0),(1 1,1 1.5,1.33333333333333 2,1 2,2 2,2 1.14285714285714,1.75 1,1 1),(3 1.71428571428571,3 2,3.5 2,3 1.71428571428571),(4 4,4 6,4.5 6,5.5 4,4 4))
+SELECT AsText(ST_SYMDIFFERENCE(
+MultiLineStringFromText('MULTILINESTRING((7 7, 1 7, 8 5, 7 8, 7 7),
+                                                 (6 3, 3 4, 1 1, 9 9, 9 0, 8 4, 9 9))'),
+Envelope(GeometryFromText('MULTIPOINT(7 9, 0 0, 3 7, 1 6, 0 0)'))));
+AsText(ST_SYMDIFFERENCE(
+MultiLineStringFromText('MULTILINESTRING((7 7, 1 7, 8 5, 7 8, 7 7),
+                                                 (6 3, 3 4, 1 1, 9 9, 9 0, 8 4, 9 9))'),
+Envelope(GeometryFromText('MULTIPOINT(7 9, 0 0, 3 7, 1 6, 0 0)'))))
+GEOMETRYCOLLECTION(LINESTRING(9 9,8 4,9 0,9 9,7 7),POLYGON((0 0,0 9,7 9,7 0,0 0)),LINESTRING(7 5.28571428571429,8 5,7 8))

mysql-test/t/gis-precise.test

@@ -158,7 +158,7 @@ SELECT Round(ST_AREA(ST_BUFFER( ST_UNION(
          POLYGONFROMTEXT('POLYGON((7 7, 4 7, 2 9, 7 6, 7 7))')), 1)), 6);
 
 
-#buf #804259 Second assertion in Gis_geometry_collection::init_from_opresult
+#bug #804259 Second assertion in Gis_geometry_collection::init_from_opresult
 
 SELECT AsText(ST_UNION(MultiPolygonFromText('
         MULTIPOLYGON(((2 2, 2 8, 8 8, 8 2, 2 2), (4 4, 4 6, 6 6, 6 4, 4 4)),
@@ -168,3 +168,12 @@ MultiPolygonFromText(' MULTIPOLYGON(((0 0, 1 9, 4 6, 0 0)),
                      ((0 5, 3 5, 3 4, 1 4, 1 3, 3 3, 3 0, 0 0, 0 5), (1 1, 2 1, 2 2, 1 2, 1 1)),
                      ((7 7, 4 7, 6 3, 7 2, 7 7)),
                      ((0 5, 3 5, 3 4, 1 4, 1 3, 3 3, 3 0, 0 0, 0 5), (1 1, 2 1, 2 2, 1 2, 1 1))) ')));
+
+
+#bug 801217 Assertion `t1->result_range' in Gcalc_operation_reducer::end_couple
+
+SELECT AsText(ST_SYMDIFFERENCE(
+        MultiLineStringFromText('MULTILINESTRING((7 7, 1 7, 8 5, 7 8, 7 7),
+                                                 (6 3, 3 4, 1 1, 9 9, 9 0, 8 4, 9 9))'),
+        Envelope(GeometryFromText('MULTIPOINT(7 9, 0 0, 3 7, 1 6, 0 0)'))));
+

sql/gcalc_tools.cc

@@ -879,7 +879,7 @@ int Gcalc_operation_reducer::count_slice(Gcalc_scan_iterator *si)
     if (!new_t)
       return 1;
     m_fn->invert_state(pi.get_shape());
-    new_t->result_range= prev_state ^ m_fn->count();
+    new_t->result_range= ~prev_state & m_fn->count();
     new_t->next= *at_hook;
     *at_hook= new_t;
     if (new_t->result_range &&
@@ -891,12 +891,17 @@ int Gcalc_operation_reducer::count_slice(Gcalc_scan_iterator *si)
   {
     active_thread *new_t0, *new_t1;
     int fn_result;
+    const Gcalc_heap::Info *p= pi.get_pi();
+    bool line= m_fn->get_shape_kind(p->shape) == Gcalc_function::shape_line;
     if (!(new_t0= new_active_thread()) || !(new_t1= new_active_thread()))
       return 1;
     
     m_fn->invert_state(pi.get_shape());
     fn_result= m_fn->count();
-    new_t0->result_range= new_t1->result_range= prev_state ^ fn_result;
+    if (line)
+      new_t0->result_range= new_t1->result_range= ~prev_state & fn_result;
+    else
+      new_t0->result_range= new_t1->result_range= prev_state ^ fn_result;
     new_t1->next= *at_hook;
     new_t0->next= new_t1;
     *at_hook= new_t0;