Merge branch 'master' into proofing

Conflicts:
	specification/0-intro.rst

api/client-server/v1/login.yaml

@@ -33,17 +33,21 @@ paths:
             type: object
             example: |-
               {
-                "username": "cheeky_monkey",
+                "type": "m.login.pasword",
+                "user": "cheeky_monkey",
                 "password": "ilovebananas"
               }
             properties:
-              username:
+              type:
+                type: string
+                description: The login type being used. Currently only "m.login.password" is supported.
+              user:
                 type: string
                 description: The fully qualified user ID or just local part of the user ID, to log in.
               password:
                 type: string
                 description: The user's password.
-            required: ["username", "password"]
+            required: ["type", "user", "password"]
       responses:
         200:
           description: The user has been authenticated.
@@ -78,6 +82,15 @@ paths:
               home_server:
                 type: string
                 description: The hostname of the Home Server on which the account has been registered.
+        400:
+          description: |-
+            Part of the request was invalid. For example, the login type may not be recognised.
+          examples:
+            application/json: |-
+              {
+                  "errcode": "M_UNKNOWN",
+                  "error": "Bad login type."
+              }
         403:
           description: |-
             The login attempt failed. For example, the password may have been incorrect.

event-schemas/schema/v1/m.room.message

@@ -1,7 +1,7 @@
 {
     "type": "object",
     "title": "Message",
-    "description": "This event is used when sending messages in a room. Messages are not limited to be text. The ``msgtype`` key outlines the type of message, e.g. text, audio, image, video, etc. The ``body`` key is text and MUST be used with every kind of ``msgtype`` as a fallback mechanism for when a client cannot render a message.",
+    "description": "This event is used when sending messages in a room. Messages are not limited to be text. The ``msgtype`` key outlines the type of message, e.g. text, audio, image, video, etc. The ``body`` key is text and MUST be used with every kind of ``msgtype`` as a fallback mechanism for when a client cannot render a message. This allows clients to display *something* even if it is just plain text.",
     "allOf": [{
         "$ref": "core-event-schema/room_event.json"
     }],

event-schemas/schema/v1/m.room.message.feedback

@@ -1,7 +1,7 @@
 {
     "type": "object",
     "title": "MessageFeedback",
-    "description": "Feedback events are events sent to acknowledge a message in some way. There are two supported acknowledgements: ``delivered`` (sent when the event has been received) and ``read`` (sent when the event has been observed by the end-user). The ``target_event_id`` should reference the ``m.room.message`` event being acknowledged. N.B. not implemented in Synapse, and superceded in v2 CS API by the ``relates_to`` event field.",
+    "description": "**NB: Usage of this event is discouraged in favour of the** `receipts module`_. **Most clients will not recognise this event.** Feedback events are events sent to acknowledge a message in some way. There are two supported acknowledgements: ``delivered`` (sent when the event has been received) and ``read`` (sent when the event has been observed by the end-user). The ``target_event_id`` should reference the ``m.room.message`` event being acknowledged.",
     "allOf": [{
         "$ref": "core-event-schema/room_event.json"
     }],

scripts/gendoc.py

@@ -128,11 +128,8 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
                 )
             else:
                 rst = f.read()
-            if rst[-2:] != "\n\n":
+
-                raise Exception(
+            rst += "\n\n"
-                    ("File %s should end with TWO new-line characters to ensure " +
-                    "file concatenation works correctly.") % (file_info,)
-                )
             return rst
     # dicts look like {0: filepath, 1: filepath} where the key is the title level
     elif isinstance(file_info, dict):

scripts/speculator/README

@@ -8,3 +8,7 @@ It serves the following HTTP endpoints:
 
 To run it, you must install the `go` tool, and run:
  `go run main.go`
+
+To build the binary (which is necessary for deployment to the matrix.org
+servers), you must again install `go`, and then run:
+ `go build`

scripts/speculator/main.go

@@ -364,6 +364,7 @@ func main() {
 		"illicitonion":  true,
 		"Kegsay":        true,
 		"NegativeMjark": true,
+		"richvdh":       true,
 	}
 	rand.Seed(time.Now().Unix())
 	masterCloneDir, err := gitClone(matrixDocCloneURL, false)

specification/0-events.rst

@@ -26,8 +26,6 @@ prefixed with ``m.``
 
 {{m_room_create_event}}
 
-{{m_room_history_visibility_event}}
-
 {{m_room_join_rules_event}}
 
 {{m_room_member_event}}

specification/0-intro.rst

@@ -427,6 +427,7 @@ Some requests have unique error codes:
 :``M_BAD_PAGINATION``:
   Encountered when specifying bad pagination query parameters.
 
+.. _sect:txn_ids:
 
 The Client-Server API typically uses ``HTTP POST`` to submit requests. This
 means these requests are not idempotent. The C-S API also allows ``HTTP PUT`` to

specification/1-client_server_api.rst

@@ -215,6 +215,12 @@ To respond to this type, reply with an auth dict as follows::
     "password": "<password>"
   }
 
+.. WARNING::
+  Clients SHOULD enforce that the password provided is suitably complex. The
+  password SHOULD include a lower-case letter, an upper-case letter, a number
+  and a symbol and be at a minimum 8 characters in length. Servers MAY reject
+  weak passwords with an error code ``M_WEAK_PASSWORD``.
+
 Google ReCaptcha
 ++++++++++++++++
 :Type:

specification/modules/content_repo.rst

@@ -52,7 +52,17 @@ The HTTP GET endpoint does not require any authentication. Knowing the URL of
 the content is sufficient to retrieve the content, even if the entity isn't in
 the room.
 
-Homeservers have additional concerns:
+MXC URIs are vulnerable to directory traversal attacks such as
+``mxc://127.0.0.1/../../../some_service/etc/passwd``. This would cause the target
+homeserver to try to access and return this file. As such, homeservers MUST
+sanitise MXC URIs by allowing only alphanumeric (``A-Za-z0-9``), ``_``
+and  ``-`` characters in the ``server-name`` and ``media-id`` values. This set
+of whitelisted characters allows URL-safe base64 encodings specified in RFC 4648.
+Applying this character whitelist is preferable to blacklisting ``.`` and ``/``
+as there are techniques around blacklisted characters (percent-encoded characters,
+UTF-8 encoded traversals, etc).
+
+Homeservers have additional content-specific concerns:
 
  - Clients may try to upload very large files. Homeservers should not store files
    that are too large and should not serve them to clients.

specification/modules/history_visibility.rst

@@ -1,21 +1,54 @@
 Room History Visibility
------------------------
+=======================
 
 .. _module:history-visibility:
 
-Whether a member of a room can see the events that happened in a room from
+This module adds support for controlling the visibility of previous events in a
-before they joined the room is controlled by the ``history_visibility`` key
+room.
-of the ``m.room.history_visibility`` state event. The valid values for
-``history_visibility`` are:
 
-- ``shared``
+In all cases, a user needs to join a room to view events in that room. Once they
-- ``invited``
+have joined a room, they will gain access to a subset of events in the room. How
-- ``joined``
+this subset is chosen is controlled by the ``m.room.history_visibility`` event
+outlined below. After a user has left a room, they may seen any events which they
+were allowed to see before they left the room, but no events received after they
+left.
 
-By default if no ``history_visibility`` is set it is assumed to be ``shared``.
+The three options for the ``m.room.history_visibility`` event are:
 
-The rules governing whether a user is allowed to see an event depend solely on
+- ``shared`` - Previous events are always accessible to newly joined members. All
-the state of the room at that event:
+  events in the room are accessible, even those sent when the member was not a part
+  of the room.
+- ``invited`` - Previous events are accessible to newly joined members from the point
+  they were invited onwards. Events stop being accessible when the member's state
+  changes to something other than ``invite`` or ``join``.
+- ``joined`` - Previous events are accessible to newly joined members from the point
+  they joined the room onwards. Events stop being accessible when the member's state
+  changes to something other than ``join``.
+
+.. WARNING::
+  These options are applied at the point an event is *sent*. Checks are
+  performed with the state of the ``m.room.history_visibility`` event when the
+  event in question is added to the DAG. This means clients cannot
+  retrospectively choose to show or hide history to new users if the setting at
+  that time was more restrictive.
+
+Events
+------
+
+{{m_room_history_visibility_event}}
+
+Client behaviour
+----------------
+
+Clients that implement this module MUST present to the user the possible options
+for setting history visibility when creating a room. 
+
+Server behaviour
+----------------
+
+By default if no ``history_visibility`` is set the visibility is assumed to be
+``shared``. The rules governing whether a user is allowed to see an event depend
+solely on the state of the room *at that event*:
 
 1. If the user was joined, allow.
 2. If the user was invited and the ``history_visibility`` was set to
@@ -24,3 +57,10 @@ the state of the room at that event:
    was set to ``shared``, allow.
 4. Otherwise, deny.
 
+Security considerations
+-----------------------
+
+The default value for ``history_visibility`` is ``shared`` for
+backwards-compatibility reasons. Clients need to be aware that by not setting
+this event they are exposing all of their room history to anyone in the room.
+

specification/modules/instant_messaging.rst

@@ -3,6 +3,10 @@ Instant Messaging
 
 .. _module:im:
 
+This module adds support for sending human-readable messages to a room. It also
+adds support for associating human-readable information with the room itself
+such as a room name and topic.
+
 Events
 ------
 
@@ -10,20 +14,138 @@ Events
 
 {{m_room_message_feedback_event}}
 
+Usage of this event is discouraged for several reasons:
+ - The number of feedback events will grow very quickly with the number of users
+   in the room. This event provides no way to "batch" feedback, unlike the
+   `receipts module`_.
+ - Pairing feedback to messages gets complicated when paginating as feedback
+   arrives before the message it is acknowledging.
+ - There are no guarantees that the client has seen the event ID being
+   acknowledged.
+
+
+.. _`receipts module`: `module:receipts`_
+
 {{m_room_name_event}}
 
 {{m_room_topic_event}}
 
 m.room.message msgtypes
------------------------
+~~~~~~~~~~~~~~~~~~~~~~~
-
-.. TODO-spec
-   How a client should handle unknown message types.
-
 
 Each `m.room.message`_ MUST have a ``msgtype`` key which identifies the type
 of message being sent. Each type has their own required and optional keys, as
-outlined below.
+outlined below. If a client cannot display the given ``msgtype`` then it SHOULD
+display the fallback plain text ``body`` key instead.
 
 {{msgtype_events}}
 
+
+Client behaviour
+----------------
+
+Clients SHOULD verify the structure of incoming events to ensure that the
+expected keys exist and that they are of the right type. Clients can discard
+malformed events or display a placeholder message to the user. Redacted
+``m.room.message`` events MUST be removed from the client. This can either be
+replaced with placeholder text (e.g. "[REDACTED]") or the redacted message can
+be removed entirely from the messages view.
+
+Events which have attachments (e.g. ``m.image``, ``m.file``) SHOULD be
+uploaded using the `content repository module`_ where available. The
+resulting ``mxc://`` URI can then be used in the ``url`` key.
+
+.. _`content repository module`: `module:content`_
+
+Recommendations when sending messages
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Clients can send messages using ``POST`` or ``PUT`` requests. Clients SHOULD use
+``PUT`` requests with `transaction IDs`_ to make requests idempotent. This
+ensures that messages are sent exactly once even under poor network conditions.
+Clients SHOULD retry requests using an exponential-backoff algorithm for a
+certain amount of time T. It is recommended that T is no longer than 5 minutes.
+After this time, the client should stop retrying and mark the message as "unsent".
+Users should be able to manually resend unsent messages.
+
+Users may type several messages at once and send them all in quick succession.
+Clients SHOULD preserve the order in which they were sent by the user. This
+means that clients should wait for the response to the previous request before
+sending the next request. This can lead to head-of-line blocking. In order to
+reduce the impact of head-of-line blocking, clients should use a queue per room
+rather than a global queue, as ordering is only relevant within a single room
+rather than between rooms.
+
+.. _`transaction IDs`: `sect:txn_ids`_
+
+Local echo
+~~~~~~~~~~
+
+Messages SHOULD appear immediately in the message view when a user presses the
+"send" button. This should occur even if the message is still sending. This is
+referred to as "local echo". Clients SHOULD implement "local echo" of messages.
+Clients MAY display messages in a different format to indicate that the server
+has not processed the message. This format should be removed when the server
+responds.
+
+Clients need to be able to match the message they are sending with the same
+message which they receive from the event stream. The echo of the same message
+from the event stream is referred to as "remote echo". Both echoes need to be
+identified as the same message in order to prevent duplicate messages being
+displayed. Ideally this pairing would occur transparently to the user: the UI
+would not flicker as it transitions from local to remote. Flickering cannot be
+fully avoided in the current client-server API. Two scenarios need to be
+considered:
+
+- The client sends a message and the remote echo arrives on the event stream
+  *after* the request to send the message completes.
+- The client sends a message and the remote echo arrives on the event stream
+  *before* the request to send the message completes.
+
+In the first scenario, the client will receive an event ID when the request to
+send the message completes. This ID can be used to identify the duplicate event
+when it arrives on the event stream. However, in the second scenario, the event
+arrives before the client has obtained an event ID. This makes it impossible to
+identify it as a duplicate event. This results in the client displaying the
+message twice for a fraction of a second before the the original request to send
+the message completes. Once it completes, the client can take remedial actions
+to remove the duplicate event by looking for duplicate event IDs. A future version
+of the client-server API will resolve this by attaching the transaction ID of the
+sending request to the event itself.
+
+
+Displaying membership information with messages
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Clients may wish to show the display name and avatar URL of the room member who
+sent a message. This can be achieved by inspecting the ``m.room.member`` state
+event for that user ID.
+
+When a user paginates the message history, clients may wish to show the
+**historical** display name and avatar URL for a room member. This is possible
+because older ``m.room.member`` events are returned when paginating. This can
+be implemented efficiently by keeping two sets of room state: old and current.
+As new events arrive and/or the user paginates back in time, these two sets of
+state diverge from each other. New events update the current state and paginated
+events update the old state. When paginated events are processed sequentially,
+the old state represents the state of the room *at the time the event was sent*.
+This can then be used to set the historical display name and avatar URL.
+
+Server behaviour
+----------------
+
+Homeservers SHOULD reject ``m.room.message`` events which don't have a
+``msgtype`` key, or which don't have a textual ``body`` key, with an HTTP status
+code of 400.
+
+Security considerations
+-----------------------
+
+Messages sent using this module are not encrypted. Messages can be encrypted
+using the `E2E module`_.
+
+Clients should sanitise **all displayed keys** for unsafe HTML to prevent Cross-Site
+Scripting (XSS) attacks. This includes room names and topics.
+
+.. _`E2E module`: `module:e2e`_
+

specification/modules/push.rst

@@ -353,14 +353,14 @@ Examples
 To create a rule that suppresses notifications for the room with ID
 ``!dj234r78wl45Gh4D:matrix.org``::
 
-  curl -X PUT -H "Content-Type: application/json" "http://localhost:8008/_matrix/client/api/v1/pushrules/global/room/%21dj234r78wl45Gh4D%3Amatrix.org?access_token=123456" -d \
+  curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/room/%21dj234r78wl45Gh4D%3Amatrix.org?access_token=123456" -d \
   '{
      "actions" : ["dont_notify"]
    }'
 
 To suppress notifications for the user ``@spambot:matrix.org``::
 
-  curl -X PUT -H "Content-Type: application/json" "http://localhost:8008/_matrix/client/api/v1/pushrules/global/sender/%40spambot%3Amatrix.org?access_token=123456" -d \
+  curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/sender/%40spambot%3Amatrix.org?access_token=123456" -d \
   '{
      "actions" : ["dont_notify"]
    }'
@@ -368,7 +368,7 @@ To suppress notifications for the user ``@spambot:matrix.org``::
 To always notify for messages that contain the work 'cake' and set a specific
 sound (with a rule_id of ``SSByZWFsbHkgbGlrZSBjYWtl``)::
 
-  curl -X PUT -H "Content-Type: application/json" "http://localhost:8008/_matrix/client/api/v1/pushrules/global/content/SSByZWFsbHkgbGlrZSBjYWtl?access_token=123456" -d \
+  curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/content/SSByZWFsbHkgbGlrZSBjYWtl?access_token=123456" -d \
   '{
      "pattern": "cake",
      "actions" : ["notify", {"set_sound":"cakealarm.wav"}]
@@ -377,7 +377,7 @@ sound (with a rule_id of ``SSByZWFsbHkgbGlrZSBjYWtl``)::
 To add a rule suppressing notifications for messages starting with 'cake' but
 ending with 'lie', superseding the previous rule::
 
-  curl -X PUT -H "Content-Type: application/json" "http://localhost:8008/_matrix/client/api/v1/pushrules/global/content/U3BvbmdlIGNha2UgaXMgYmVzdA?access_token=123456&before=SSByZWFsbHkgbGlrZSBjYWtl" -d \
+  curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/content/U3BvbmdlIGNha2UgaXMgYmVzdA?access_token=123456&before=SSByZWFsbHkgbGlrZSBjYWtl" -d \
   '{
      "pattern": "cake*lie",
      "actions" : ["notify"]
@@ -387,7 +387,7 @@ To add a custom sound for notifications messages containing the word 'beer' in
 any rooms with 10 members or fewer (with greater importance than the room,
 sender and content rules)::
 
-  curl -X PUT -H "Content-Type: application/json" "http://localhost:8008/_matrix/client/api/v1/pushrules/global/override/U2VlIHlvdSBpbiBUaGUgRHVrZQ?access_token=123456" -d \
+  curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/v1/pushrules/global/override/U2VlIHlvdSBpbiBUaGUgRHVrZQ?access_token=123456" -d \
   '{
      "conditions": [
        {"kind": "event_match", "key": "content.body", "pattern": "beer" },