From 3d8f637e555ec7061e9d7582caf9c518050973d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Barrag=C3=A1n=20Merino?= Date: Wed, 16 Nov 2016 13:00:09 +0100 Subject: [PATCH] Fix json_to_jsonb migrations and prevent \u0000 error --- .../migrations/0011_json_to_jsonb.py | 8 +-- .../history/migrations/0014_json_to_jsonb.py | 34 +++++++++--- .../projects/migrations/0055_json_to_jsonb.py | 53 +++++++++++++++---- .../timeline/migrations/0006_json_to_jsonb.py | 2 +- taiga/users/migrations/0023_json_to_jsonb.py | 2 +- .../migrations/0003_json_to_jsonb.py | 2 +- .../webhooks/migrations/0006_json_to_jsonb.py | 14 +++-- 7 files changed, 87 insertions(+), 28 deletions(-) diff --git a/taiga/projects/custom_attributes/migrations/0011_json_to_jsonb.py b/taiga/projects/custom_attributes/migrations/0011_json_to_jsonb.py index e9a17bcc..cfe4ff4c 100644 --- a/taiga/projects/custom_attributes/migrations/0011_json_to_jsonb.py +++ b/taiga/projects/custom_attributes/migrations/0011_json_to_jsonb.py @@ -17,7 +17,7 @@ class Migration(migrations.Migration): ALTER TABLE "{table_name}" ALTER COLUMN "{column_name}" TYPE jsonb - USING to_json("{column_name}"::text)::jsonb; + USING regexp_replace("{column_name}"::text, '\\u0000', '\\\\u0000', 'g')::jsonb; """.format( table_name="custom_attributes_epiccustomattributesvalues", column_name="attributes_values", @@ -29,7 +29,7 @@ class Migration(migrations.Migration): ALTER TABLE "{table_name}" ALTER COLUMN "{column_name}" TYPE jsonb - USING to_json("{column_name}"::text)::jsonb; + USING regexp_replace("{column_name}"::text, '\\u0000', '\\\\u0000', 'g')::jsonb; """.format( table_name="custom_attributes_userstorycustomattributesvalues", column_name="attributes_values", @@ -41,7 +41,7 @@ class Migration(migrations.Migration): ALTER TABLE "{table_name}" ALTER COLUMN "{column_name}" TYPE jsonb - USING to_json("{column_name}"::text)::jsonb; + USING regexp_replace("{column_name}"::text, '\\u0000', '\\\\u0000', 'g')::jsonb; """.format( table_name="custom_attributes_taskcustomattributesvalues", column_name="attributes_values", @@ -53,7 +53,7 @@ class Migration(migrations.Migration): ALTER TABLE "{table_name}" ALTER COLUMN "{column_name}" TYPE jsonb - USING to_json("{column_name}"::text)::jsonb; + USING regexp_replace("{column_name}"::text, '\\u0000', '\\\\u0000', 'g')::jsonb; """.format( table_name="custom_attributes_issuecustomattributesvalues", column_name="attributes_values", diff --git a/taiga/projects/history/migrations/0014_json_to_jsonb.py b/taiga/projects/history/migrations/0014_json_to_jsonb.py index 7906f0ac..c0704233 100644 --- a/taiga/projects/history/migrations/0014_json_to_jsonb.py +++ b/taiga/projects/history/migrations/0014_json_to_jsonb.py @@ -16,13 +16,33 @@ class Migration(migrations.Migration): migrations.RunSQL( """ ALTER TABLE "history_historyentry" - ALTER COLUMN "delete_comment_user" TYPE jsonb USING to_json("delete_comment_user"::text)::jsonb, - ALTER COLUMN "comment_versions" TYPE jsonb USING to_json("comment_versions"::text)::jsonb, - ALTER COLUMN "values_diff_cache" TYPE jsonb USING to_json("values_diff_cache"::text)::jsonb, - ALTER COLUMN "user" TYPE jsonb USING to_json("user"::text)::jsonb, - ALTER COLUMN "diff" TYPE jsonb USING to_json("diff"::text)::jsonb, - ALTER COLUMN "snapshot" TYPE jsonb USING to_json("snapshot"::text)::jsonb, - ALTER COLUMN "values" TYPE jsonb USING to_json("values"::text)::jsonb; + ALTER COLUMN "delete_comment_user" + TYPE jsonb + USING regexp_replace("delete_comment_user"::text, '\\u0000', '\\\\u0000', 'g')::jsonb, + + ALTER COLUMN "comment_versions" + TYPE jsonb + USING regexp_replace("comment_versions"::text, '\\u0000', '\\\\u0000', 'g')::jsonb, + + ALTER COLUMN "values_diff_cache" + TYPE jsonb + USING regexp_replace("values_diff_cache"::text, '\\u0000', '\\\\u0000', 'g')::jsonb, + + ALTER COLUMN "user" + TYPE jsonb + USING regexp_replace("user"::text, '\\u0000', '\\\\u0000', 'g')::jsonb, + + ALTER COLUMN "diff" + TYPE jsonb + USING regexp_replace("diff"::text, '\\u0000', '\\\\u0000', 'g')::jsonb, + + ALTER COLUMN "snapshot" + TYPE jsonb + USING regexp_replace("snapshot"::text, '\\u0000', '\\\\u0000', 'g')::jsonb, + + ALTER COLUMN "values" + TYPE jsonb + USING regexp_replace("values"::text, '\\u0000', '\\\\u0000', 'g')::jsonb; """, reverse_sql=migrations.RunSQL.noop ), diff --git a/taiga/projects/migrations/0055_json_to_jsonb.py b/taiga/projects/migrations/0055_json_to_jsonb.py index 5b9a0b8e..4530bf7f 100644 --- a/taiga/projects/migrations/0055_json_to_jsonb.py +++ b/taiga/projects/migrations/0055_json_to_jsonb.py @@ -16,23 +16,54 @@ class Migration(migrations.Migration): migrations.RunSQL( """ ALTER TABLE "projects_projectmodulesconfig" - ALTER COLUMN "config" TYPE jsonb USING to_json("config"::text)::jsonb; + ALTER COLUMN "config" + TYPE jsonb + USING regexp_replace("config"::text, '\\u0000', '\\\\u0000', 'g')::jsonb; """, reverse_sql=migrations.RunSQL.noop ), migrations.RunSQL( """ ALTER TABLE "projects_projecttemplate" - ALTER COLUMN "roles" TYPE jsonb USING to_json("roles"::text)::jsonb, - ALTER COLUMN "default_options" TYPE jsonb USING to_json("default_options"::text)::jsonb, - ALTER COLUMN "epic_statuses" TYPE jsonb USING to_json("epic_statuses"::text)::jsonb, - ALTER COLUMN "us_statuses" TYPE jsonb USING to_json("us_statuses"::text)::jsonb, - ALTER COLUMN "points" TYPE jsonb USING to_json("points"::text)::jsonb, - ALTER COLUMN "task_statuses" TYPE jsonb USING to_json("task_statuses"::text)::jsonb, - ALTER COLUMN "issue_statuses" TYPE jsonb USING to_json("issue_statuses"::text)::jsonb, - ALTER COLUMN "issue_types" TYPE jsonb USING to_json("issue_types"::text)::jsonb, - ALTER COLUMN "priorities" TYPE jsonb USING to_json("priorities"::text)::jsonb, - ALTER COLUMN "severities" TYPE jsonb USING to_json("severities"::text)::jsonb; + ALTER COLUMN "roles" + TYPE jsonb + USING regexp_replace("roles"::text, '\\u0000', '\\\\u0000', 'g')::jsonb, + + ALTER COLUMN "default_options" + TYPE jsonb + USING regexp_replace("default_options"::text, '\\u0000', '\\\\u0000', 'g')::jsonb, + + ALTER COLUMN "epic_statuses" + TYPE jsonb + USING regexp_replace("epic_statuses"::text, '\\u0000', '\\\\u0000', 'g')::jsonb, + + ALTER COLUMN "us_statuses" + TYPE jsonb + USING regexp_replace("us_statuses"::text, '\\u0000', '\\\\u0000', 'g')::jsonb, + + ALTER COLUMN "points" + TYPE jsonb + USING regexp_replace("points"::text, '\\u0000', '\\\\u0000', 'g')::jsonb, + + ALTER COLUMN "task_statuses" + TYPE jsonb + USING regexp_replace("task_statuses"::text, '\\u0000', '\\\\u0000', 'g')::jsonb, + + ALTER COLUMN "issue_statuses" + TYPE jsonb + USING regexp_replace("issue_statuses"::text, '\\u0000', '\\\\u0000', 'g')::jsonb, + + ALTER COLUMN "issue_types" + TYPE jsonb + USING regexp_replace("issue_types"::text, '\\u0000', '\\\\u0000', 'g')::jsonb, + + ALTER COLUMN "priorities" + TYPE jsonb + USING regexp_replace("priorities"::text, '\\u0000', '\\\\u0000', 'g')::jsonb, + + ALTER COLUMN "severities" + TYPE jsonb + USING regexp_replace("severities"::text, '\\u0000', '\\\\u0000', 'g')::jsonb; """, reverse_sql=migrations.RunSQL.noop ), diff --git a/taiga/timeline/migrations/0006_json_to_jsonb.py b/taiga/timeline/migrations/0006_json_to_jsonb.py index 72a63968..b6b17e54 100644 --- a/taiga/timeline/migrations/0006_json_to_jsonb.py +++ b/taiga/timeline/migrations/0006_json_to_jsonb.py @@ -18,7 +18,7 @@ class Migration(migrations.Migration): ALTER TABLE "{table_name}" ALTER COLUMN "{column_name}" TYPE jsonb - USING to_json("{column_name}"::text)::jsonb; + USING regexp_replace("{column_name}"::text, '\\u0000', '\\\\u0000', 'g')::jsonb; """.format( table_name="timeline_timeline", column_name="data", diff --git a/taiga/users/migrations/0023_json_to_jsonb.py b/taiga/users/migrations/0023_json_to_jsonb.py index 34782897..f16d5699 100644 --- a/taiga/users/migrations/0023_json_to_jsonb.py +++ b/taiga/users/migrations/0023_json_to_jsonb.py @@ -18,7 +18,7 @@ class Migration(migrations.Migration): ALTER TABLE "{table_name}" ALTER COLUMN "{column_name}" TYPE jsonb - USING to_json("{column_name}"::text)::jsonb; + USING regexp_replace("{column_name}"::text, '\\u0000', '\\\\u0000', 'g')::jsonb; """.format( table_name="users_authdata", column_name="extra", diff --git a/taiga/userstorage/migrations/0003_json_to_jsonb.py b/taiga/userstorage/migrations/0003_json_to_jsonb.py index 9e1f5243..228324ac 100644 --- a/taiga/userstorage/migrations/0003_json_to_jsonb.py +++ b/taiga/userstorage/migrations/0003_json_to_jsonb.py @@ -18,7 +18,7 @@ class Migration(migrations.Migration): ALTER TABLE "{table_name}" ALTER COLUMN "{column_name}" TYPE jsonb - USING to_json("{column_name}"::text)::jsonb; + USING regexp_replace("{column_name}"::text, '\\u0000', '\\\\u0000', 'g')::jsonb; """.format( table_name="userstorage_storageentry", column_name="value", diff --git a/taiga/webhooks/migrations/0006_json_to_jsonb.py b/taiga/webhooks/migrations/0006_json_to_jsonb.py index 9f40b860..ed6258e3 100644 --- a/taiga/webhooks/migrations/0006_json_to_jsonb.py +++ b/taiga/webhooks/migrations/0006_json_to_jsonb.py @@ -16,9 +16,17 @@ class Migration(migrations.Migration): migrations.RunSQL( """ ALTER TABLE "webhooks_webhooklog" - ALTER COLUMN "request_headers" TYPE jsonb USING to_json("request_headers"::text)::jsonb, - ALTER COLUMN "request_data" TYPE jsonb USING to_json("request_data"::text)::jsonb, - ALTER COLUMN "response_headers" TYPE jsonb USING to_json("response_headers"::text)::jsonb; + ALTER COLUMN "request_headers" + TYPE jsonb + USING regexp_replace("request_headers"::text, '\\u0000', '\\\\u0000', 'g')::jsonb, + + ALTER COLUMN "request_data" + TYPE jsonb + USING regexp_replace("request_data"::text, '\\u0000', '\\\\u0000', 'g')::jsonb, + + ALTER COLUMN "response_headers" + TYPE jsonb + USING regexp_replace("response_headers"::text, '\\u0000', '\\\\u0000', 'g')::jsonb; """, reverse_sql=migrations.RunSQL.noop ),