Talk:External services description

About the different alternatives describing the available WS, by Eloy

Hi, after reading the different alternatives, I'm inclined to implement something like the 3rd one. Having our custom structures to define the interface to the available web services seems really better than using "free-form" structures like "raw" arrays and objects or using harcoded keywords like "multiple" and so on. It has various advantages and doesn't hurt simplicity when defining the parameters:

Better PHP structures <--> XML mapping
Extensibility:
- custom/reusable validation methods.
- built-in XML writing
- enforce some sort of common response functionality (see my concerns about returned content below).

Anyway, I've some (small) concerns about the example code used in the 3rd alternative, so I'm writing here how it should look like ideally IMO (rationale about changes is after the code):

class moodle_group_external extends external_api {

   public static function add_member_parameters() {
       return new external_single_structure(
           'member', array(
               new external_param('groupid', PARAM_INT, 'some group id'),
               new external_param('userid', PARAM_INT, 'some user id')
           )
       );
   }
   public static function add_member($groupid, $userid) {
       $params = self::validate_prameters(self::add_member_parameters(), array('groupid'=>$groupid, 'userid'=>$userid));

       // all the parameter/behavioural checks and security constrainsts go here,
       // throwing exceptions if neeeded and and calling low level (grouplib)
       // add_member() function that will be one in charge of the functionality without
       // further checks.

   }
   public static function add_member_returns() { // Some concerns here. See below.
       return null;
   }

   public static function add_members_parameters() {
       return new external_multiple_structure(
           'membership', self::add_member_parameters()
       );
   }
   public static function add_members(array $membership) {
       foreach($membership as $one) { // simply one iterator over the "single" function if possible
           self::add_member($one->groupid, $one->userid);
       }
   }
   public static function add_members_returns() { // BIG concerns here. See below.
       return null;
   }

   public static function get_groups_parameters() {
       return new external_multiple_structure(
           'groups', new external_single_structure(
               'group', array(
                   new external_param('groupid', PARAM_INT, 'some group id')
               )
           )
       );
   }
   public static function get_groups(array $groups) {
       $params = self::validate_prameters(self::get_groups_parameters(), array('groups'=>$groups));

       // all the parameter/behavioural checks and security constrainsts go here,
       // throwing exceptions if neeeded and and calling low level (grouplib)
       // get_groups() function that will be one in charge of the functionality without
       // further checks.

   }
   public static function get_groups_returns() { // BIG concerns here. See below.
       return new external_multiple_structure(
           'groups', new external_single_structure(
               'group', array(
                   new external_param('id', PARAM_INT, 'some group id'),
                   new external_param('name', PARAM_TEXT, 'multilang compatible name, course unique'),
                   new external_param('description', PARAM_RAW, 'just some text'),
                   new external_param('enrolmentkey', PARAM_RAW, 'group enrol secret phrase')
               )
           )
       );
   }

}


class moodle_user_external extends external_api {

   public static function create_users_parameters() {
       return new external_multiple_structure(
           'users', new external_single_structure(
               'user', array(
                   new external_param('username', PARAM_USERNAME, 'Username policy is defined in Moodle security config'),
                   new external_param('password', PARAM_RAW, 'Moodle passwords can consist of any character'),
                   new external_param('firstname', PARAM_NOTAGS, 'The first name(s) of the user'),
                   new external_param('lastname', PARAM_NOTAGS, 'The family name of the user'),
                   new external_param('email', PARAM_EMAIL, 'A valid and unique email address'),
                   new external_param('auth', PARAM_AUTH, 'Auth plugins include manual, ldap, imap, etc', false),
                   new external_param('confirmed', PARAM_NUMBER, 'Active user: 1 if confirmed, 0 otherwise', false),
                   new external_param('idnumber', PARAM_RAW, 'An arbitrary ID code number perhaps from the institution', false),
                   new external_param('emailstop', PARAM_NUMBER, 'Email is blocked: 1 is blocked and 0 otherwise', false),
                   new external_param('lang', PARAM_LANG, 'Language code such as "en_utf8", must exist on server', false),
                   new external_param('theme', PARAM_THEME, 'Theme name such as "standard", must exist on server', false),
                   new external_param('timezone', PARAM_ALPHANUMEXT, 'Timezone code such as Australia/Perth, or 99 for default', false),
                   new external_param('mailformat', PARAM_INTEGER, 'Mail format code is 0 for plain text, 1 for HTML etc', false),
                   new external_param('description', PARAM_TEXT, 'User profile description, as HTML', false),
                   new external_param('city', PARAM_NOTAGS, 'Home city of the user', false),
                   new external_param('country', PARAM_ALPHA, 'Home country code of the user, such as AU or CZ', false),
                   new external_multiple_structure(
                       'preferences', new external_single_structure(
                           'preference', array(
                               new external_param('type', PARAM_ALPHANUMEXT, 'The name of the preference'),
                               new external_param('value', PARAM_RAW, 'The value of the preference')
                           )
                       ), 'User preferences', false),
                   new external_multiple_structure(
                       'customfields', new external_single_structure(
                           'customfield', array(
                               new external_param('type', PARAM_ALPHANUMEXT, 'The name of the custom field'),
                               new external_param('value', PARAM_RAW, 'The value of the custom field')
                           )
                       ), 'User custom fields', false)
               )
           )
       );
   }
   public static function create_users(array $users) {
       $params = self::validate_prameters(self::create_users_parameters(), array('users'=>$users));

       foreach ($users as $user) {
           // all the parameter/behavioural checks and security constrainsts go here,
           // throwing exceptions if neeeded and and calling low level (userlib)
           // add_user() function that will be one in charge of the functionality without
           // further checks.
       }
   }
   public static function create_users_returns() { // BIG concerns here. See below.
       return new external_multiple_structure(
           'users', new external_single_structure(
               'user', array(
                   new external_param('userid', PARAM_INT, 'id of the created user'))
           )
       );
   }

}

Changes from original code are:

Small change in structure names. external_bulk_array renamed to external_multiple_structure and external_assoc_array to external_single_structure (note that structure can be changed to anything else). The important bits are the single/multiple ones vs. the assoc/bulk. Clear IMO.
Moved the "name" of the single structure and param classes to the constructor (no need for associative arrays). Clear syntax.
Small change in external_multiple_structure to add the missing "name" to its constructor. Matching previous point.
The xxxx_parameters() function will return directly one single or multiple structure, saving one nesting level from original code.
Reuse as much as possible:
- The moodle_group_external is a clear example. Once we have the single service implemented (add_member), it's pretty easy to build its multiple alternative. Just reuse the single parameters definition and the single executor. IMO we should always enforce the single/multiple duality.
- Also, although there isn't an example, I think we could have some "habitual" structures predefined somewhere, see for example, the preference or the customfield single structures (or the whole preferences or customfields multiple ones if you prefer). They are basically the same (name/value pairs). It could be useful to have one predefined "name_value" structure somewhere and use it where necessary (user preferences, user custom fields, various config options...).

Concerns about return functions:

In the example above we have a good example. There are some xxxx_return() functions returning NULLs, others returning multiple structures (and also is possible to return single structures or numbers or whatever).

IMO returned structures should be, always, encapsulated into something constant, call it 'external_response'. Then, within that, we can return some constant information (bool result, string error, whatever...) and some dynamic information (single/multiple structures).

Also, while it's really simple to return results for single structures (for example the add_member() above will return one simple result ), it's more complex to do so for multiple structures. For example, the add_members() above must return one different result for each groupid-userid added. Imagine you create one client sending 4 pairs (within a multiple structure well formed WS request).

And only one of them is added successfully, other fails because the userid is incorrect, another fails because it already exists and the 4th fails because the user cannot handle that group, or because the user isn't enrolled in the course the group belongs to or... whatever. How will the return information be formatted to inform about all those heterogeneous situations? How will the client know the exact status of each pair sent in the multiple request?

Should we introduce one mandatory "key" parameter in all the occurrences within one multiple request, in order to get the response properly identified by those keys? Or will the multiple request be rejected as a whole (I think this isn't possible).

Note I don't know if that has been resolved in other places of the WS documentation but I haven't been able to find it, hence the warn. ;-)

So, summarising, I like the 3rd alternative with the changes specified above (in order to make it more readable/simple, basically). And I think the "response" (return) part is at least as important as the "request" (parameters) part, and it still needs more work/definition/polishing. Shouldn't be difficult to achieve that.

Hope this helps, ciao --Eloy Lafuente (stronk7) 00:24, 18 September 2009 (UTC)

Offtopic: I hate the "external" keyword everywhere. :-D :-P

Note: The second revision to this alternative (adding new "name" parameter to all the classes vs old associative arrays) has been discussed/agreed with Petr. --Eloy Lafuente (stronk7) 20:39, 19 September 2009 (UTC)

I think the flexible return values are problematic - I personally prefer exception when anything goes wrong, getting complex result requires extra post processing of results. So my -1 here. Petr Škoda (škoďák)

But "client" must know what has been processed and what not. What if something breaks in the 4th element of a multiple request? How is the client informed about the 3 first elements already being processed and the rest not? Eloy Lafuente (stronk7) 09:10, 18 September 2009 (UTC)

Right, the methods working with bulk data should do that, I am just not sure single methods should return complex structures, also we could send this extra information through exception - the bulk methods should call simple ones and could return exception containing the original exception + progress info Petr Škoda (škoďák) 20:32, 19 September 2009 (UTC)

I agree with that (the bulk function will catch exception of the single function, and keep going to process the rest) Jerome Mouneyrac (jmouneyrac) 12:23, 21 September 2009 (Perth)

Agree too, that is part of code reusability (throwing exception with useful info in single/multiple methods and catching it at higher level to provide WS response. Anyway my concerns are about to have a well-defined response, able to support responses both to single and multiple requests. Right now, it's undefined. --Eloy Lafuente (stronk7) 07:51, 21 September 2009 (UTC)

the parameter syntax allows us to describe both simple and complex return types, for any kind of problems we would use exceptions - in fact we could define add 3rd description function which describes some basic exceptions - I do not like the idea of using normal return when something goes wrong

Validate_parameters requires that we define a way to match parameters with the description

Edit: I consider that external functions are normal php functions. So they can accept anything as parameters (very complex param like array of object containing array of object), and they can have several parameters.

I'm writing the validate_parameters function. There is a thing we didn't define into the description specs.
In order to validate parameters, we need to be able to match the parameters to the description
I guess we'll match the parameters like that :
description => parameter type
external_multiple_structure => a non associative array
external_single_structure => an object

Moreover I think funtion_name_parameters() should return a array of external_xxxx_structure. So we can support functions with more than one parameter like function_name($param1, $param2, $param3).
The validation call will be $params = self::validate_prameters(self::function_name_description(), array($param1, $param2, $param3));

oh no - we can use multiple params of PHP function, the mapping is simple, you just need to solve the order of params in PHP to names in description - my proposal was solving exactly that Petr Škoda (škoďák) 15:37, 24 September 2009 (UTC)

///example for a function with one param only

 public static function create_users_parameters() {
       return array (
           new external_multiple_structure(
       'users', new external_single_structure(
       'user', array(
       new external_param('username', PARAM_USERNAME, 'Username policy is defined in Moodle security config'),
       new external_param('password', PARAM_RAW, 'Moodle passwords can consist of any character'),
       new external_param('firstname', PARAM_NOTAGS, 'The first name(s) of the user'),
       new external_param('lastname', PARAM_NOTAGS, 'The family name of the user'),
       new external_param('email', PARAM_EMAIL, 'A valid and unique email address'),
       new external_param('auth', PARAM_AUTH, 'Auth plugins include manual, ldap, imap, etc', false),
       new external_param('confirmed', PARAM_NUMBER, 'Active user: 1 if confirmed, 0 otherwise', false),
       new external_param('idnumber', PARAM_RAW, 'An arbitrary ID code number perhaps from the institution', false),
       new external_param('emailstop', PARAM_NUMBER, 'Email is blocked: 1 is blocked and 0 otherwise', false),
       new external_param('lang', PARAM_LANG, 'Language code such as "en_utf8", must exist on server', false),
       new external_param('theme', PARAM_THEME, 'Theme name such as "standard", must exist on server', false),
       new external_param('timezone', PARAM_ALPHANUMEXT, 'Timezone code such as Australia/Perth, or 99 for default', false),
       new external_param('mailformat', PARAM_INTEGER, 'Mail format code is 0 for plain text, 1 for HTML etc', false),
       new external_param('description', PARAM_TEXT, 'User profile description, as HTML', false),
       new external_param('city', PARAM_NOTAGS, 'Home city of the user', false),
       new external_param('country', PARAM_ALPHA, 'Home country code of the user, such as AU or CZ', false),
       new external_multiple_structure(
       'preferences', new external_single_structure(
       'preference', array(
       new external_param('type', PARAM_ALPHANUMEXT, 'The name of the preference'),
       new external_param('value', PARAM_RAW, 'The value of the preference')
       )
       ), 'User preferences', false),
       new external_multiple_structure(
       'customfields', new external_single_structure(
       'customfield', array(
       new external_param('type', PARAM_ALPHANUMEXT, 'The name of the custom field'),
       new external_param('value', PARAM_RAW, 'The value of the custom field')
       )
       ), 'User custom fields', false)
       )
       )
       )
       ///if the function had a second param we would add it here
       /// , new external_multiple_structure(...)
       );
   }

public static function create_users(array $users) {
       ...
       $params = self::validate_prameters(self::create_users_parameters(), array($users));

/// if the function had a second param we would have wrote
/// public static function create_users(array $users, array $second_param) {
/// $params = self::validate_prameters(self::create_users_parameters(), array($users, $second_param));

Integrate file download into the Moodle web services framework

Overview

Currently (Moodle 2.3) web service client developers cannot download files without having to create extra code outside of the web services framework. For example, in order to develop a web service function that downloads student submission files for an assignment they have to:

Create a web service function that performs the business logic and puts the files in a location for downloading, named something like mod_assign_movesubmissionfilestoalocationfordownloading(assignid)
Download the file using webservices/pluginfile.php

This 2 step process is cumbersome for clients who want a web service to consist solely of web service functions without the need to call additional php code (webservices/pluginfile.php) outside of the framework.

The changes that we propose will allow the above 2 step process to be replaced by a single web service function call that efficiently returns a file (i.e. NOT encoding in the XML but using http streaming like pluginfile.php). So, the 2 step process described above would be replaced by a single web service function:

mod_assign_getsubmissionfiles(assignid)

Description of proposed changes

We have created modifications to the Moodle web services by adding a new external type called external_file to allow direct file download. Our changes can be accessed from http://tracker.moodle.org/browse/MDL-32336. Our changes modify the web services servers and complement the existing web services so that the list of return data types available to developers will be:

external_value
external_single_structure
external_multiple_structure
external_file

In MDL-32336 we have made our changes to the REST server. In the case of the other protocols we have the option to use either the same implementation or an implementation that is specific to the protocol. For instance, in the case of SOAP we could implement the file download using SOAP with attachments (http://www.w3.org/TR/SOAP-attachments). The simplest option, and our preference, is to use the same implementation for all servers.

The following code snippet shows the new proposed class external_file that is added to lib/externallib.php:

/**

* File description class. Which indicates the response will return a file for download.
* Implementation that is using external_file is expected to return an associative-array/hashmap
* which contains the file to be downloaded.
* 
* The file download process is handled using send_file() and send_temp_file() from lib/filelib.php.
* The properties of the associative-array/hashmap expected are:
* - filename  - string - proposed file name when saving file
* - path      - string - path to file, or content of file itself
* - tempfile  - boolean - will use send_temp_file() if true, otherwise use send_file()
*/

class external_file extends external_description {

   public function __construct($desc=, $required=VALUE_REQUIRED, $default=null) {
       parent::__construct($desc, $required, $default);
   }

}

The following snippet describes a simple web service function which downloads a file using external_file type.

class mod_assign_external extends external_api {

   public static function getsubmissionfiles_parameters() {
       return new external_function_parameters(
               array('assignid' => new external_value(PARAM_INT, 'id of assignment for which submission files will be downloaded')));
   }
   
   public static function getsubmissionfiles($assignid) {

       // perform application logic, get files and package them in zipfilename
       
       $result = array();
       $result['filename'] = $zipfilename;
       $result['path'] = '/somepath' . $zipfilename;
       $result['tempfile'] = true;
       
       return $result;
   }
   
   public static function getsubmissionfiles_returns() {
       return new external_file();
   }

}

Added supported param types

Added a list of supported parameters types as of version 2.3 (from "moodlelib.php"), I think it can be useful for developers working with web-services.

--Camilo Rivera 02:51, 22 November 2012 (WST)

Documentation