Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Reduce unnecessary whitespace in JSON #7372

Merged
merged 15 commits into from
Aug 7, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/7372.misc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Reduce the amount of whitespace in JSON stored and sent in responses. Contributed by David Vo.
5 changes: 3 additions & 2 deletions synapse/http/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
from typing import Any, Callable, Dict, Tuple, Union

import jinja2
from canonicaljson import encode_canonical_json, encode_pretty_printed_json, json
from canonicaljson import encode_canonical_json, encode_pretty_printed_json

from twisted.internet import defer
from twisted.python import failure
Expand All @@ -46,6 +46,7 @@
from synapse.http.site import SynapseRequest
from synapse.logging.context import preserve_fn
from synapse.logging.opentracing import trace_servlet
from synapse.util import json_encoder
from synapse.util.caches import intern_dict

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -538,7 +539,7 @@ def respond_with_json(
# canonicaljson already encodes to bytes
json_bytes = encode_canonical_json(json_object)
else:
json_bytes = json.dumps(json_object).encode("utf-8")
json_bytes = json_encoder.encode(json_object).encode("utf-8")

return respond_with_json_bytes(request, code, json_bytes, send_cors=send_cors)

Expand Down
5 changes: 3 additions & 2 deletions synapse/replication/tcp/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,12 @@
allowed to be sent by which side.
"""
import abc
import json
import logging
from typing import Tuple, Type

_json_encoder = json.JSONEncoder()
from canonicaljson import json

from synapse.util import json_encoder as _json_encoder

logger = logging.getLogger(__name__)

Expand Down
4 changes: 2 additions & 2 deletions synapse/rest/media/v1/preview_url_resource.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
from urllib import parse as urlparse

import attr
from canonicaljson import json

from twisted.internet import defer
from twisted.internet.error import DNSLookupError
Expand All @@ -43,6 +42,7 @@
from synapse.logging.context import make_deferred_yieldable, run_in_background
from synapse.metrics.background_process_metrics import run_as_background_process
from synapse.rest.media.v1._base import get_filename_from_headers
from synapse.util import json_encoder
from synapse.util.async_helpers import ObservableDeferred
from synapse.util.caches.expiringcache import ExpiringCache
from synapse.util.stringutils import random_string
Expand Down Expand Up @@ -355,7 +355,7 @@ async def _do_preview(self, url: str, user: str, ts: int) -> bytes:

logger.debug("Calculated OG for %s as %s", url, og)

jsonog = json.dumps(og)
jsonog = json_encoder.encode(og)

# store OG in history-aware DB cache
await self.store.store_url_cache(
Expand Down
7 changes: 3 additions & 4 deletions synapse/storage/databases/main/account_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,12 @@
import logging
from typing import List, Tuple

from canonicaljson import json

from twisted.internet import defer

from synapse.storage._base import SQLBaseStore, db_to_json
from synapse.storage.database import DatabasePool
from synapse.storage.util.id_generators import StreamIdGenerator
from synapse.util import json_encoder
from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
from synapse.util.caches.stream_change_cache import StreamChangeCache

Expand Down Expand Up @@ -327,7 +326,7 @@ def add_account_data_to_room(self, user_id, room_id, account_data_type, content)
Returns:
A deferred that completes once the account_data has been added.
"""
content_json = json.dumps(content)
content_json = json_encoder.encode(content)

with self._account_data_id_gen.get_next() as next_id:
# no need to lock here as room_account_data has a unique constraint
Expand Down Expand Up @@ -373,7 +372,7 @@ def add_account_data_for_user(self, user_id, account_data_type, content):
Returns:
A deferred that completes once the account_data has been added.
"""
content_json = json.dumps(content)
content_json = json_encoder.encode(content)

with self._account_data_id_gen.get_next() as next_id:
# no need to lock here as account_data has a unique constraint on
Expand Down
9 changes: 4 additions & 5 deletions synapse/storage/databases/main/deviceinbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,12 @@
import logging
from typing import List, Tuple

from canonicaljson import json

from twisted.internet import defer

from synapse.logging.opentracing import log_kv, set_tag, trace
from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause
from synapse.storage.database import DatabasePool
from synapse.util import json_encoder
from synapse.util.caches.expiringcache import ExpiringCache

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -354,7 +353,7 @@ def add_messages_txn(txn, now_ms, stream_id):
)
rows = []
for destination, edu in remote_messages_by_destination.items():
edu_json = json.dumps(edu)
edu_json = json_encoder.encode(edu)
rows.append((destination, stream_id, now_ms, edu_json))
txn.executemany(sql, rows)

Expand Down Expand Up @@ -432,7 +431,7 @@ def _add_messages_to_local_device_inbox_txn(
# Handle wildcard device_ids.
sql = "SELECT device_id FROM devices WHERE user_id = ?"
txn.execute(sql, (user_id,))
message_json = json.dumps(messages_by_device["*"])
message_json = json_encoder.encode(messages_by_device["*"])
for row in txn:
# Add the message for all devices for this user on this
# server.
Expand All @@ -454,7 +453,7 @@ def _add_messages_to_local_device_inbox_txn(
# Only insert into the local inbox if the device exists on
# this server
device = row[0]
message_json = json.dumps(messages_by_device[device])
message_json = json_encoder.encode(messages_by_device[device])
messages_json_for_user[device] = message_json

if messages_json_for_user:
Expand Down
11 changes: 5 additions & 6 deletions synapse/storage/databases/main/devices.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@
import logging
from typing import List, Optional, Set, Tuple

from canonicaljson import json

from twisted.internet import defer

from synapse.api.errors import Codes, StoreError
Expand All @@ -36,6 +34,7 @@
make_tuple_comparison_clause,
)
from synapse.types import Collection, get_verify_key_from_cross_signing_key
from synapse.util import json_encoder
from synapse.util.caches.descriptors import (
Cache,
cached,
Expand Down Expand Up @@ -397,7 +396,7 @@ def _add_user_signature_change_txn(self, txn, from_user_id, user_ids, stream_id)
values={
"stream_id": stream_id,
"from_user_id": from_user_id,
"user_ids": json.dumps(user_ids),
"user_ids": json_encoder.encode(user_ids),
},
)

Expand Down Expand Up @@ -1032,7 +1031,7 @@ def _update_remote_device_list_cache_entry_txn(
txn,
table="device_lists_remote_cache",
keyvalues={"user_id": user_id, "device_id": device_id},
values={"content": json.dumps(content)},
values={"content": json_encoder.encode(content)},
# we don't need to lock, because we assume we are the only thread
# updating this user's devices.
lock=False,
Expand Down Expand Up @@ -1088,7 +1087,7 @@ def _update_remote_device_list_cache_txn(self, txn, user_id, devices, stream_id)
{
"user_id": user_id,
"device_id": content["device_id"],
"content": json.dumps(content),
"content": json_encoder.encode(content),
}
for content in devices
],
Expand Down Expand Up @@ -1209,7 +1208,7 @@ def _add_device_outbound_poke_to_stream_txn(
"device_id": device_id,
"sent": False,
"ts": now,
"opentracing_context": json.dumps(context)
"opentracing_context": json_encoder.encode(context)
if whitelisted_homeserver(destination)
else "{}",
}
Expand Down
11 changes: 5 additions & 6 deletions synapse/storage/databases/main/e2e_room_keys.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from canonicaljson import json

from twisted.internet import defer

from synapse.api.errors import StoreError
from synapse.logging.opentracing import log_kv, trace
from synapse.storage._base import SQLBaseStore, db_to_json
from synapse.util import json_encoder


class EndToEndRoomKeyStore(SQLBaseStore):
Expand Down Expand Up @@ -50,7 +49,7 @@ def update_e2e_room_key(self, user_id, version, room_id, session_id, room_key):
"first_message_index": room_key["first_message_index"],
"forwarded_count": room_key["forwarded_count"],
"is_verified": room_key["is_verified"],
"session_data": json.dumps(room_key["session_data"]),
"session_data": json_encoder.encode(room_key["session_data"]),
},
desc="update_e2e_room_key",
)
Expand All @@ -77,7 +76,7 @@ def add_e2e_room_keys(self, user_id, version, room_keys):
"first_message_index": room_key["first_message_index"],
"forwarded_count": room_key["forwarded_count"],
"is_verified": room_key["is_verified"],
"session_data": json.dumps(room_key["session_data"]),
"session_data": json_encoder.encode(room_key["session_data"]),
}
)
log_kv(
Expand Down Expand Up @@ -360,7 +359,7 @@ def _create_e2e_room_keys_version_txn(txn):
"user_id": user_id,
"version": new_version,
"algorithm": info["algorithm"],
"auth_data": json.dumps(info["auth_data"]),
"auth_data": json_encoder.encode(info["auth_data"]),
},
)

Expand All @@ -387,7 +386,7 @@ def update_e2e_room_keys_version(
updatevalues = {}

if info is not None and "auth_data" in info:
updatevalues["auth_data"] = json.dumps(info["auth_data"])
updatevalues["auth_data"] = json_encoder.encode(info["auth_data"])
if version_etag is not None:
updatevalues["etag"] = version_etag

Expand Down
5 changes: 3 additions & 2 deletions synapse/storage/databases/main/end_to_end_keys.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,15 @@
# limitations under the License.
from typing import Dict, List, Tuple

from canonicaljson import encode_canonical_json, json
from canonicaljson import encode_canonical_json

from twisted.enterprise.adbapi import Connection
from twisted.internet import defer

from synapse.logging.opentracing import log_kv, set_tag, trace
from synapse.storage._base import SQLBaseStore, db_to_json
from synapse.storage.database import make_in_list_sql_clause
from synapse.util import json_encoder
from synapse.util.caches.descriptors import cached, cachedList
from synapse.util.iterutils import batch_iter

Expand Down Expand Up @@ -700,7 +701,7 @@ def _set_e2e_cross_signing_key_txn(self, txn, user_id, key_type, key):
values={
"user_id": user_id,
"keytype": key_type,
"keydata": json.dumps(key),
"keydata": json_encoder.encode(key),
"stream_id": stream_id,
},
)
Expand Down
5 changes: 2 additions & 3 deletions synapse/storage/databases/main/event_push_actions.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,10 @@
import logging
from typing import List

from canonicaljson import json

from synapse.metrics.background_process_metrics import run_as_background_process
from synapse.storage._base import LoggingTransaction, SQLBaseStore, db_to_json
from synapse.storage.database import DatabasePool
from synapse.util import json_encoder
from synapse.util.caches.descriptors import cachedInlineCallbacks

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -50,7 +49,7 @@ def _serialize_action(actions, is_highlight):
else:
if actions == DEFAULT_NOTIF_ACTION:
return ""
return json.dumps(actions)
return json_encoder.encode(actions)


def _deserialize_action(actions, is_highlight):
Expand Down
17 changes: 8 additions & 9 deletions synapse/storage/databases/main/group_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,11 @@

from typing import List, Tuple

from canonicaljson import json

from twisted.internet import defer

from synapse.api.errors import SynapseError
from synapse.storage._base import SQLBaseStore, db_to_json
from synapse.util import json_encoder

# The category ID for the "default" category. We don't store as null in the
# database to avoid the fun of null != null
Expand Down Expand Up @@ -752,7 +751,7 @@ def upsert_group_category(self, group_id, category_id, profile, is_public):
if profile is None:
insertion_values["profile"] = "{}"
else:
update_values["profile"] = json.dumps(profile)
update_values["profile"] = json_encoder.encode(profile)

if is_public is None:
insertion_values["is_public"] = True
Expand Down Expand Up @@ -783,7 +782,7 @@ def upsert_group_role(self, group_id, role_id, profile, is_public):
if profile is None:
insertion_values["profile"] = "{}"
else:
update_values["profile"] = json.dumps(profile)
update_values["profile"] = json_encoder.encode(profile)

if is_public is None:
insertion_values["is_public"] = True
Expand Down Expand Up @@ -1007,7 +1006,7 @@ def _add_user_to_group_txn(txn):
"group_id": group_id,
"user_id": user_id,
"valid_until_ms": remote_attestation["valid_until_ms"],
"attestation_json": json.dumps(remote_attestation),
"attestation_json": json_encoder.encode(remote_attestation),
},
)

Expand Down Expand Up @@ -1131,7 +1130,7 @@ def _register_user_group_membership_txn(txn, next_id):
"is_admin": is_admin,
"membership": membership,
"is_publicised": is_publicised,
"content": json.dumps(content),
"content": json_encoder.encode(content),
},
)

Expand All @@ -1143,7 +1142,7 @@ def _register_user_group_membership_txn(txn, next_id):
"group_id": group_id,
"user_id": user_id,
"type": "membership",
"content": json.dumps(
"content": json_encoder.encode(
{"membership": membership, "content": content}
),
},
Expand Down Expand Up @@ -1171,7 +1170,7 @@ def _register_user_group_membership_txn(txn, next_id):
"group_id": group_id,
"user_id": user_id,
"valid_until_ms": remote_attestation["valid_until_ms"],
"attestation_json": json.dumps(remote_attestation),
"attestation_json": json_encoder.encode(remote_attestation),
},
)
else:
Expand Down Expand Up @@ -1240,7 +1239,7 @@ def update_remote_attestion(self, group_id, user_id, attestation):
keyvalues={"group_id": group_id, "user_id": user_id},
updatevalues={
"valid_until_ms": attestation["valid_until_ms"],
"attestation_json": json.dumps(attestation),
"attestation_json": json_encoder.encode(attestation),
},
desc="update_remote_attestion",
)
Expand Down
Loading