April 2004 - MPlayer-G2-dev - lists.mplayerhq.hu

Experimental vo_gl with (limited) support for hardware YUV.
by Reimar Döffinger 23 Apr '04

23 Apr '04

Hi, just thought I let everyone have a look at what I'm currently working on. It can play YUV directly on nVidia GPUs better than GeForce3, but probably won't work at all with older cards :( - it's still a long way till it's finished, but I hope the concept will be extensible enough. On my PC it uses slightly less CPU even than xv (even with sync-to-vblank enabled). As always comments or test results are welcome. Greetings, Reimar Döffinger #include <stdio.h> #include <stdlib.h> #include <errno.h> #include "config.h" #include "mp_msg.h" #include "video_out.h" #include "video/vfcap.h" #include "video/img_format.h" #include <string.h> #include <X11/Xlib.h> #include <X11/Xutil.h> #include <GL/gl.h> #include <GL/glext.h> #include <GL/glx.h> #define DEFAULT_INTERNAL_FORMAT GL_RGB typedef enum { VO_GL_NO_CONV, VO_GL_REG_COMB_CONV, VO_GL_FRAG_PROG_CONV } glConvType; struct vo_buffer_priv_s { GLuint planeTexture[4]; GLuint glDisplayList; }; struct vo_priv_s { Window window; Display *display; int img_width; int img_height; int d_width; int d_height; char numPlanes; GLint planeInternalFormat[4]; GLsizei planeTextureWidth[4]; GLsizei planeTextureHeight[4]; int planeWidth[4]; int planeHeight[4]; char planeBytesPerPixel[4]; GLenum planeFormat[4]; GLenum planeType[4]; glConvType conversion; char supportYUV; char useGL12; XVisualInfo vinfo; GLXContext context; glConvType bestYUVConversion; void (APIENTRY * glCombinerParameterfv) (GLenum, const GLfloat *); void (APIENTRY * glCombinerParameteri) (GLenum, GLint); void (APIENTRY * glCombinerInput) (GLenum, GLenum, GLenum, GLenum, GLenum, GLenum); void (APIENTRY * glCombinerOutput) (GLenum, GLenum, GLenum, GLenum, GLenum, GLenum, GLenum, GLboolean, GLboolean, GLboolean); void (APIENTRY * glActiveTexture) (GLenum); void (APIENTRY * glMultiTexCoord2f) (GLenum, GLfloat, GLfloat); }; static void resize (vo_instance_t * vo, int src[4]) { mp_msg (MSGT_VO, MSGL_V, "[gl] resize "); mp_msg (MSGT_VO, MSGL_V, "(%i, %i, %i, %i),", vo->x, vo->y, vo->w, vo->h); if (src) mp_msg (MSGT_VO, MSGL_V, "(%i, %i, %i, %i)\n", src[0], src[1], src[2], src[3]); else mp_msg (MSGT_VO, MSGL_V, "NULL\n"); glViewport (vo->x, vo->y, vo->w, vo->h); if (src) { glMatrixMode (GL_PROJECTION); glLoadIdentity (); // FIXME: how do we find out when we should flip ?? glTranslatef (-src[0], -src[1], 0); glOrtho (0, src[2], src[3], 0, -1, 1); glMatrixMode (GL_MODELVIEW); glLoadIdentity (); } } static void getGLFunctions (vo_instance_t * vo) { GLint maxGeneralCombiners; GLint maxTextureUnits; vo->priv->glCombinerParameterfv = glXGetProcAddressARB ("glCombinerParameterfvNV"); vo->priv->glCombinerParameteri = glXGetProcAddressARB ("glCombinerParameteriNV"); vo->priv->glCombinerInput = glXGetProcAddressARB ("glCombinerInputNV"); vo->priv->glCombinerOutput = glXGetProcAddressARB ("glCombinerOutputNV"); vo->priv->glActiveTexture = glXGetProcAddressARB ("glActiveTextureARB"); vo->priv->glMultiTexCoord2f = glXGetProcAddressARB ("glMultiTexCoord2fARB"); glGetIntegerv (GL_MAX_GENERAL_COMBINERS_NV, &maxGeneralCombiners); glGetIntegerv (GL_MAX_TEXTURE_UNITS, &maxTextureUnits); if (maxTextureUnits >= 3 && maxGeneralCombiners >= 2) vo->priv->bestYUVConversion = VO_GL_REG_COMB_CONV; else vo->priv->bestYUVConversion = VO_GL_NO_CONV; } static int control (vo_instance_t * vo, int request, void *data, ...) { switch (request) { case VOCTRL_SET_WINDOW: vo->priv->window = *((Window *) data); if (!vo->priv->context) { vo->priv->context = glXCreateContext (vo->priv->display, &vo->priv->vinfo, NULL, True); } else glFinish (); glXMakeCurrent (vo->priv->display, vo->priv->window, vo->priv->context); return VO_TRUE; case VOCTRL_MOVE_DEST: vo->x = ((int *) data)[0]; vo->y = ((int *) data)[1]; resize (vo, NULL); return VO_TRUE; case VOCTRL_RESIZE_DEST: vo->w = ((int *) data)[0]; vo->h = ((int *) data)[1]; resize (vo, NULL); return VO_TRUE; case VOCTRL_RESIZE_SRC: resize (vo, data); return VO_TRUE; } return VO_NOTIMPL; } static unsigned int query_format (vo_instance_t * vo, unsigned int format, int width, int height) { const int supported = VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW | VFCAP_HWSCALE_UP | VFCAP_HWSCALE_DOWN | VFCAP_FLIP; switch (format) { case IMGFMT_RGB8: case IMGFMT_RGB15: case IMGFMT_RGB16: case IMGFMT_RGB24: case IMGFMT_RGB32: case IMGFMT_BGR8: case IMGFMT_BGR15: case IMGFMT_BGR16: case IMGFMT_BGR24: case IMGFMT_BGR32: case IMGFMT_Y800: case IMGFMT_YV12: return supported; } return 0; } static void uninit (vo_instance_t * vo) { if (vo->priv->context) glXDestroyContext (vo->priv->display, vo->priv->context); free (vo->priv); free (vo); } typedef struct { int pixelBytes; GLint internalFormat; GLenum format; GLenum type; int width_div; int height_div; } PlaneInfo; typedef struct { unsigned int format; char numPlanes; PlaneInfo plane[4]; } FormatInfo; FormatInfo formatInfos[] = { { IMGFMT_RGB8, 1, { {1, 0, GL_RGB, GL_UNSIGNED_BYTE_2_3_3_REV, 1, 1}, } }, { IMGFMT_RGB15, 1, { {2, 0, GL_RGBA, GL_UNSIGNED_SHORT_1_5_5_5_REV, 1, 1}, } }, { IMGFMT_RGB16, 1, { {2, 0, GL_RGB, GL_UNSIGNED_SHORT_5_6_5_REV, 1, 1}, } }, { IMGFMT_RGB24, 1, { {3, 0, GL_RGB, GL_UNSIGNED_BYTE, 1, 1}, } }, { IMGFMT_RGB32, 1, { {4, 0, GL_RGBA, GL_UNSIGNED_BYTE, 1, 1}, } }, { IMGFMT_BGR8, 1, { {1, 0, GL_RGB, GL_UNSIGNED_BYTE_3_3_2, 1, 1}, } }, { IMGFMT_BGR15, 1, { {2, 0, GL_BGRA, GL_UNSIGNED_SHORT_1_5_5_5_REV, 1, 1}, } }, { IMGFMT_BGR16, 1, { {2, 0, GL_RGB, GL_UNSIGNED_SHORT_5_6_5, 1, 1}, } }, { IMGFMT_BGR24, 1, { {3, 0, GL_BGR, GL_UNSIGNED_BYTE, 1, 1}, } }, { IMGFMT_BGR32, 1, { {4, 0, GL_BGRA, GL_UNSIGNED_BYTE, 1, 1}, } }, { IMGFMT_Y800, 1, { {1, GL_LUMINANCE, GL_LUMINANCE, GL_UNSIGNED_BYTE, 1, 1}, } }, { IMGFMT_YV12, 3, { {1, GL_LUMINANCE, GL_LUMINANCE, GL_UNSIGNED_BYTE, 1, 1}, {1, GL_LUMINANCE, GL_LUMINANCE, GL_UNSIGNED_BYTE, 2, 2}, {1, GL_LUMINANCE, GL_LUMINANCE, GL_UNSIGNED_BYTE, 2, 2}, } }, { IMGFMT_444P, 3, { {1, GL_LUMINANCE, GL_LUMINANCE, GL_UNSIGNED_BYTE, 1, 1}, {1, GL_LUMINANCE, GL_LUMINANCE, GL_UNSIGNED_BYTE, 1, 1}, {1, GL_LUMINANCE, GL_LUMINANCE, GL_UNSIGNED_BYTE, 1, 1}, } }, { 0, 0, { {0, 0, 0, 0, 0, 0}, } } }; static void analyzeFormat (vo_instance_t * vo, unsigned int format) { int i = 0; FormatInfo *fi; while (formatInfos[i].format != 0 && formatInfos[i].format != format) i++; if (formatInfos[i].format != format) { mp_msg (MSGT_VO, MSGL_ERR, "[gl] Unknown image format. This shouldn't happen!\n"); }; fi = &formatInfos[i]; vo->priv->numPlanes = fi->numPlanes; for (i = 0; i < fi->numPlanes; i++) { vo->priv->planeWidth[i] = vo->priv->img_width / fi->plane[i].width_div; vo->priv->planeHeight[i] = vo->priv->img_height / fi->plane[i].height_div; vo->priv->planeBytesPerPixel[i] = fi->plane[i].pixelBytes; if (fi->plane[i].internalFormat != 0) vo->priv->planeInternalFormat[i] = fi->plane[i].internalFormat; else vo->priv->planeInternalFormat[i] = DEFAULT_INTERNAL_FORMAT; vo->priv->planeFormat[i] = fi->plane[i].format; vo->priv->planeType[i] = fi->plane[i].type; } } static void findTextureSizes (vo_instance_t * vo) { int i, tmp; for (i = 0; i < vo->priv->numPlanes; i++) { tmp = 1; while (tmp < vo->priv->planeWidth[i]) tmp *= 2; vo->priv->planeTextureWidth[i] = tmp; tmp = 1; while (tmp < vo->priv->planeHeight[i]) tmp *= 2; vo->priv->planeTextureHeight[i] = tmp; } } static void createTextures (vo_instance_t * vo, vo_buffer_t * buf) { int i; char *texInitData = NULL; char initVal = 0; GLfloat borderColor[4]; int size; glGenTextures (vo->priv->numPlanes, buf->priv->planeTexture); for (i = 0; i < vo->priv->numPlanes; i++) { glActiveTexture (GL_TEXTURE0 + i); glEnable (GL_TEXTURE_2D); glBindTexture (GL_TEXTURE_2D, buf->priv->planeTexture[i]); glTexParameterf (GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameterf (GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); mp_msg (MSGT_VO, MSGL_V, "[gl] Allocating texture of size %dx%d ...\n", vo->priv->planeTextureWidth[i], vo->priv->planeTextureHeight[i]); size = vo->priv->planeTextureWidth[i] * vo->priv->planeTextureHeight[i]; texInitData = malloc (size); if (i == 0) { borderColor[0] = 0.0; borderColor[1] = 0.0; borderColor[2] = 0.0; borderColor[3] = 0.0; initVal = 0; } else { // no green borders in YUV borderColor[0] = 0.5; borderColor[1] = 0.5; borderColor[2] = 0.5; borderColor[3] = 0.0; initVal = 128; } memset (texInitData, initVal, size); glTexImage2D (GL_TEXTURE_2D, 0, vo->priv->planeInternalFormat[i], vo->priv->planeTextureWidth[i], vo->priv->planeTextureHeight[i], 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, texInitData); mp_msg (MSGT_VO, MSGL_V, "[gl] glTexImage2D finished with %d\n", glGetError ()); free (texInitData); glTexParameterfv (GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR, borderColor); } } static void setupConversion (vo_instance_t * vo) { int i; GLfloat ucoef[] = { 0, -0.344, 1.770, 0 }; GLfloat vcoef[] = { 1.403, -0.714, 0, 0 }; switch (vo->priv->conversion) { case VO_GL_NO_CONV: break; case VO_GL_REG_COMB_CONV: // It seems that coefficient must be in [0,1] range, so here comes // the trick: // first put them in [-0.5, 0.5] range, then add 0.5; // can be undone with the HALF_BIAS and SCALE_BY_FOUR arguments // for CombinerInput and CombinerOutput for (i = 0; i < 4; i++) { ucoef[i] = ucoef[i] * 0.25 + 0.5; vcoef[i] = vcoef[i] * 0.25 + 0.5; } vo->priv->glCombinerParameterfv (GL_CONSTANT_COLOR0_NV, ucoef); vo->priv->glCombinerParameterfv (GL_CONSTANT_COLOR1_NV, vcoef); // UV first, like this green component can't overflow vo->priv->glCombinerInput (GL_COMBINER0_NV, GL_RGB, GL_VARIABLE_A_NV, GL_TEXTURE1, GL_HALF_BIAS_NORMAL_NV, GL_RGB); vo->priv->glCombinerInput (GL_COMBINER0_NV, GL_RGB, GL_VARIABLE_B_NV, GL_CONSTANT_COLOR0_NV, GL_HALF_BIAS_NORMAL_NV, GL_RGB); vo->priv->glCombinerInput (GL_COMBINER0_NV, GL_RGB, GL_VARIABLE_C_NV, GL_TEXTURE2, GL_HALF_BIAS_NORMAL_NV, GL_RGB); vo->priv->glCombinerInput (GL_COMBINER0_NV, GL_RGB, GL_VARIABLE_D_NV, GL_CONSTANT_COLOR1_NV, GL_HALF_BIAS_NORMAL_NV, GL_RGB); vo->priv->glCombinerOutput (GL_COMBINER0_NV, GL_RGB, GL_DISCARD_NV, GL_DISCARD_NV, GL_SPARE0_NV, GL_SCALE_BY_FOUR_NV, GL_NONE, GL_FALSE, GL_FALSE, GL_FALSE); vo->priv->glCombinerInput (GL_COMBINER1_NV, GL_RGB, GL_VARIABLE_A_NV, GL_SPARE0_NV, GL_SIGNED_IDENTITY_NV, GL_RGB); vo->priv->glCombinerInput (GL_COMBINER1_NV, GL_RGB, GL_VARIABLE_B_NV, GL_ZERO, GL_UNSIGNED_INVERT_NV, GL_RGB); vo->priv->glCombinerInput (GL_COMBINER1_NV, GL_RGB, GL_VARIABLE_C_NV, GL_TEXTURE0, GL_SIGNED_IDENTITY_NV, GL_RGB); vo->priv->glCombinerInput (GL_COMBINER1_NV, GL_RGB, GL_VARIABLE_D_NV, GL_ZERO, GL_UNSIGNED_INVERT_NV, GL_RGB); vo->priv->glCombinerOutput (GL_COMBINER1_NV, GL_RGB, GL_DISCARD_NV, GL_DISCARD_NV, GL_SPARE0_NV, GL_NONE, GL_NONE, GL_FALSE, GL_FALSE, GL_FALSE); // leave final combiner stage in default mode vo->priv->glCombinerParameteri (GL_NUM_GENERAL_COMBINERS_NV, 2); break; case VO_GL_FRAG_PROG_CONV: break; default: mp_msg (MSGT_VO, MSGL_ERR, "[gl] unknown conversion type!"); break; } } static void activateConversion (vo_instance_t * vo, vo_buffer_t * buf) { glActiveTexture (GL_TEXTURE0); glBindTexture (GL_TEXTURE_2D, buf->priv->planeTexture[0]); switch (vo->priv->conversion) { case VO_GL_NO_CONV: break; case VO_GL_REG_COMB_CONV: vo->priv->glActiveTexture (GL_TEXTURE0); glBindTexture (GL_TEXTURE_2D, buf->priv->planeTexture[0]); vo->priv->glActiveTexture (GL_TEXTURE1); glBindTexture (GL_TEXTURE_2D, buf->priv->planeTexture[2]); vo->priv->glActiveTexture (GL_TEXTURE2); glBindTexture (GL_TEXTURE_2D, buf->priv->planeTexture[1]); glEnable (GL_REGISTER_COMBINERS_NV); break; case VO_GL_FRAG_PROG_CONV: break; default: mp_msg (MSGT_VO, MSGL_ERR, "[gl] unknown conversion type!"); break; } } static void deactivateConversion (vo_instance_t * vo, vo_buffer_t * buf) { switch (vo->priv->conversion) { case VO_GL_NO_CONV: break; case VO_GL_REG_COMB_CONV: glDisable (GL_REGISTER_COMBINERS_NV); break; case VO_GL_FRAG_PROG_CONV: break; default: mp_msg (MSGT_VO, MSGL_ERR, "[gl] unknown conversion type!"); break; } } static void createDisplayList (vo_instance_t * vo, vo_buffer_t * buf) { int i; GLfloat xcov[4], ycov[4]; buf->priv->glDisplayList = glGenLists (1); for (i = 0; i < vo->priv->numPlanes; i++) { xcov[i] = (GLfloat) vo->priv->planeWidth[i] / (GLfloat) vo->priv->planeTextureWidth[i]; ycov[i] = (GLfloat) vo->priv->planeHeight[i] / (GLfloat) vo->priv->planeTextureHeight[i]; } // TODO: set this at an appropriate place to a useable default if (vo->priv->numPlanes == 3) vo->priv->conversion = vo->priv->bestYUVConversion; setupConversion (vo); glNewList (buf->priv->glDisplayList, GL_COMPILE); activateConversion (vo, buf); glBegin (GL_QUADS); for (i = 0; i < vo->priv->numPlanes; i++) glMultiTexCoord2f (GL_TEXTURE0 + i, 0, 0); glVertex2i (0, 0); for (i = 0; i < vo->priv->numPlanes; i++) glMultiTexCoord2f (GL_TEXTURE0 + i, xcov[i], 0); glVertex2i (vo->priv->img_width, 0); for (i = 0; i < vo->priv->numPlanes; i++) glMultiTexCoord2f (GL_TEXTURE0 + i, xcov[i], ycov[i]); glVertex2i (vo->priv->img_width, vo->priv->img_height); for (i = 0; i < vo->priv->numPlanes; i++) glMultiTexCoord2f (GL_TEXTURE0 + i, 0, ycov[i]); glVertex2i (0, vo->priv->img_height); glEnd (); deactivateConversion (vo, buf); glEndList (); } static int config (vo_instance_t * vo, int width, int height, int d_width, int d_height, unsigned int format) { int dest_coords[] = { 0, 0, width, height }; vo->priv->img_width = width; vo->priv->img_height = height; vo->priv->d_width = d_width; vo->priv->d_height = d_height; vo->buffer_w = width; vo->buffer_h = height; vo->w = d_width; vo->h = d_height; getGLFunctions (vo); analyzeFormat (vo, format); findTextureSizes (vo); glDisable (GL_BLEND); glDisable (GL_DEPTH_TEST); glDepthMask (GL_FALSE); glDisable (GL_CULL_FACE); glClearColor (0, 0, 0, 0); resize (vo, dest_coords); return VO_TRUE; } static void updateTexture (vo_instance_t * vo, vo_buffer_t * buf, int planeNr, int x, int y, int w, int h, unsigned char *data) { mp_msg (MSGT_VO, MSGL_DBG1, "[gl] updateTexture (%i, %i, %i, %i, %i, %x)\n", planeNr, x, y, w, h, data); glBindTexture (GL_TEXTURE_2D, buf->priv->planeTexture[planeNr]); glTexSubImage2D (GL_TEXTURE_2D, 0, x, y, w, h, vo->priv->planeFormat[planeNr], vo->priv->planeType[planeNr], data); mp_msg (MSGT_VO, MSGL_DBG1, "[gl] glTexSubImage2D finished with %d\n", glGetError ()); } static void draw_slice (vo_instance_t * vo, vo_buffer_t * buf, unsigned char *src[], int stride[], int w, int h, int x, int y) { mp_msg (MSGT_VO, MSGL_DBG1, "[gl] draw_slice\n"); } static int frame_done (vo_instance_t * vo, vo_buffer_t * buf) { int i; mp_msg (MSGT_VO, MSGL_DBG1, "[gl] frame_done\n"); for (i = 0; i < vo->priv->numPlanes; i++) updateTexture (vo, buf, i, 0, 0, vo->priv->planeWidth[i], vo->priv->planeHeight[i], buf->planes[i]); glFlush (); return 0; } static void show_frame (vo_instance_t * vo, vo_buffer_t * buf, unsigned int time_pos) { mp_msg (MSGT_VO, MSGL_DBG1, "[gl] show_frame\n"); glCallList (buf->priv->glDisplayList); glXSwapBuffers (vo->priv->display, vo->priv->window); glClear (GL_COLOR_BUFFER_BIT); } static vo_buffer_t *get_buffer (vo_instance_t * vo) { int i; vo_buffer_t *buffer = calloc (sizeof (vo_buffer_t), 1); buffer->priv = calloc (sizeof (struct vo_buffer_priv_s), 1); createTextures (vo, buffer); createDisplayList (vo, buffer); buffer->status = 0; for (i = 0; i < vo->priv->numPlanes; i++) { buffer->planes[i] = calloc (vo->priv->planeWidth[i] * vo->priv->planeHeight[i] * vo->priv->planeBytesPerPixel[i], 1); buffer->stride[i] = vo->priv->planeWidth[i] * vo->priv->planeBytesPerPixel[i]; } return buffer; } static void release_buffer (vo_instance_t * vo, vo_buffer_t * buf) { int i; for (i = 0; i < vo->priv->numPlanes; i++) free (buf->planes[i]); glDeleteLists (buf->priv->glDisplayList, 1); glDeleteTextures (vo->priv->numPlanes, buf->priv->planeTexture); free (buf->priv); free (buf); } static vo_instance_t *preinit (char *args, void *x11_display) { vo_instance_t *vo; if (!x11_display) return NULL; // X11 not available vo = calloc (sizeof (vo_instance_t), 1); vo->priv = calloc (sizeof (struct vo_priv_s), 1); memset (vo->priv, 0, sizeof (struct vo_priv_s)); vo->priv->useGL12 = 1; vo->priv->supportYUV = 0; vo->priv->display = x11_display; XMatchVisualInfo (vo->priv->display, DefaultScreen (vo->priv->display), 24, TrueColor, &vo->priv->vinfo); #if 0 if (arg) { if (strcmp (arg, "gl11") == 0) { useGL12 = 0; } else if (strcmp (arg, "glyuv") == 0) { supportYUV = 1; } else { printf ("[gl] Unknown subdevice: %s\n", arg); return ENOSYS; } } #endif vo->buffer_flags = 0; vo->num_buffers = 0; // functions: vo->query_format = query_format; vo->control = control; vo->config = config; vo->draw_slice = draw_slice; vo->frame_done = frame_done; vo->show_frame = show_frame; vo->uninit = uninit; vo->get_buffer = get_buffer; vo->release_buffer = release_buffer; return vo; } vo_info_t vo_driver_gl = { MODULE_TYPE_VO, MODULE_VERSION, preinit, "gl", "OpenGL driver", "Reimar Döffinger", "Reimar Döffinger", NULL, VO_MODULE_CAPS_X11 | VO_MODULE_CAPS_WINDOW | VO_MODULE_CAPS_GUI, NULL, NULL, 0 };

1 0

An open, standard video/audio API?
by Mikhail Ramendik 22 Apr '04

22 Apr '04

Hello, (SUMMARY. A general-purpose standard API for all multimedia processing, including codecs, filters, demuxing etc., would be very useful in GNU/Linux, and resolve many existing problems, including licensing. The MPlayer team is the best source for a standard that could be widely adopted. I would volunteer to do tech writing work on the spec). (DISCLAIMER. The following may need a sanity check; i.e. I may have lost contact with reality. If so please tell me where). I'm a user of mplayer, and I like it a big darn lot, so thanks people. (I know it's not exactly the right list for this but I had to put it in my first message to an mplayer devel list). I'm also a big fan of Open Standards. Free Software too - but while Free Software would totally rule in an ideal world, Open Standards totally rule *now*, they make the Internet run. They also encourage the development of Free Software, and minimize the effects of non-free software. And the one thing that I feel is missing in [GNU/]Linux, while present (although crippled) in the Non-Free OS, is a clearly defined standard API for all multimedia stuff. A Free Software masterpiece, VirtualDub, is made possible by DirectShow. A clear (hmm, theoretically) interface to all possible codecs, the AVI file format, and some other stuff is just already there. Yes, it's stinky (the 4GB limit, and of course the lack of freedom). But GNU/Linux has none whatsoever. Video4Linux2 contains some Codec API, but who uses it? As a result, a GNU/Linux program that wants to use codecs, filters, etc, has to care about interaction with every library out there. Look at transcode, which had to actually include a copy of ffmpeg. (And no, one can not just forget everything else and go ffmpeg only). >From the discussion I saw here, it seems that while the MPLayer G2 core could be a sort of "clearing center" for multimedia processing under GNU/Linux, it will still have a pretty specific API. My question: why not aim for a standard, generic, published, well-described multimedia processing API, one that would (because of its technical benefits and open nature) far surpass DirectShow? It should work across machines (not to speak of working across very different programs). X-Windows might provide an example. And yes, I would support sockets as the main tool, and TCP sockets where possible. Among other things, this will allow interoperation of different machines, as already done in X. Just imagine: one machine to grab the source and apply pre-procesing filters, another to do the coding, and a third to stream the content, all communicating over the Open Multimedia Protocol. This would be the *easy* scenario; hard ones would include load balancing. In most cases of course the API would work on the same machine - but still provide nice separation for different tasks, so loved by the Unix world. A codec, a filter, a muxer or demuxer, a player would all communicate via socket streams. Each one easily replaceable, without micro-management from the player (common today). <Use_case> A user has installed a cool hardware MPEG5 decoder. He previously used to run MPlayer G2 for playing MPEG5 files; his brother ran XMMS G3, and used Transcode to convert MPEG5 to Theora for watching on his Zaurus handheld. Now the just replaces the primary system codec for MPEG5. Neither MPlayer nor XMMS nor Transcode will ever notice (unless they were configured to run a soecific codec by name, overriding system settings). </Use_case> Among other matters, an open socket-based standards might resolve the recent licensing issue. The problem with hardware player developers, as it seems to be, is that they just can't release all of the source to their firmware, because of licensing matters with other companies. But what if the components they use will all interact via a standard socket interface? In this way, as the GPL FAQ exlicitly says, they can use GPL-ed and non-GPL modules together! So they can use some modules from MPlayer, AND be forced to release any changes to these modules, AND be able to pay MPlayer developers for enhancing them. But if their proprietary modules use the standard protocol, they'll be able to keep them - and honor any licensing requirement with any third company. It's stricter than the LGPL. They will be forced to adhere to the protocol and use sockets, rather than access system calls in a particular library. In fact, any DRM they install will be easy to circumvent - the output will go through the standard stream protocol, so go intercept it, and whoops! we're done. If this standard is adopted widely enough, the DRM-mongers will either have to stop using all newer versions of Linux video tools altogether, or see their data intercepted and their DRM broken by the nearest hacker who knows how to read a manual for a standard. (They still have the option of preventing physical access inside the hardware, so that one will be unable to do this interception; but one can't fix that with software licenses. That would work with 100% GPL too, just keep the key values [data not code!] in a chip inside). MPlayer is widely accepted as the best multimedia player under GNU/Linux; ffmpeg/libavcodec, as the best codec library; MEncoder and transcode are the two commonly used processing solutions. Therefore, I think the MPlayer team would be the best people for developing and promoting this standard. Now for a bit of personal stuff. I am a tech writer with over 5 years of experience behind me. English is not my native language, but I am employed as a tech writer in English by a major international company. For money, I write development docs for internal-use software (it's not distributed outside of the company) - and they like the work I do. I would be willing to take up the writing of the spec. Not the technical side of the development - I don't know *nearly* enough to do that. But I can glean information from programmer-speak, list discussions, etc., and convert it into an organized, readable document. And that's what an open standard spec should be, if it's to be widely adopted. (Ultimately, why not make it an RFC?) But of course, this is useful only if the developers here are willing to go that way. To develop an open standard for interaction between multimedia processing components. And to base the development of MPlayer G2 on this standard, not an internal API. This could be a crucial step for development of GNU/Linux as a user OS. This may be an insane raving by a misguided guy. ANyway, replies will be very much appreciated :) Yours, Mikhail Ramendik Moscow, Russia

8 44

Yet another video API. :)
by Andriy N. Gritsenko 14 Apr '04

14 Apr '04

Hi! So after some discussions and rethinking whole thing, I've made this. Big thanks for Rich and Ivan - this thing was partially based on Rich's VP api, and many things from Ivan's letter (in December 2003) were very useful. Images and buffers must be managed differently. My goals was: 1) have as little API as possible 2) use direct rendering as much as possible 3) use copying as little as possible 4) do memory allocations as little as possible 5) processing slices the same way as full frames (to be possible process slices even from decoder to vo without commiting to frame) 6) avoid pushing frames at all So I mean I wanted to do all faster and with less memory. Rich asked me on IRC about how much API we need. Here is full list of APIs which we need (AFAIK): 1) modules/config API (common); 2) stream API (file, network, etc.); 3) demuxer API; 4) video API; 5) audio API; 6) OSD/subtitles/input control API; 7) muxer API. So with these APIs we will have those (sub)layers: 1) core layer (config parser, module finder/loader) - modules/config API; 2) input stream layer (file, cache, network, etc.) - stream API; 3) demuxer layer - from stream API to demuxer API; 4) video decoder layer - from demuxer API to video API; 5) audio decoder layer - from demuxer API to audio API; 6) subtitles layer - from stream or demuxer API to OSD/subtitles API; 7) video layer - video API (chains); 8) audio layer - audio API (chains); 9) video output layer - video API (end); 10) audio output layer - audio API (end); 11) video encoder layer - from video API to muxer API; 12) audio encoder layer - from audio API to muxer API; 13) muxer layer - from muxer API to stream API; Each sublayer has own specifics and own number of APIs so don't have to be messed with another. Frames that are pulled from layers 2...8 must be pushed to layers 9...13 by application to do A/V sync. Files in attachment contain video API (for using by layers 4,7,9,11, and may be used by layer 6 too). Some minor parts are still not developed yet, I'm still in doubt about them, see my comments there. :) About future of G2... I think it's good to migrate from G1 to G2 and it may be good to migrate step by step, i.e.: a) create new development branch of MPlayer (1.1 for example); b) implement modules/config API to it; c) create new demuxer layer with new demuxer API; d) develop new video layer with new video API and so on. Each step will take some time but it's much better than rewrite all at once. With best wishes. Andriy. P.S. Don't be mad at my poor English, please. :) P.P.S. All above is IMHO. :)

2 2